From antocuni at codespeak.net Sat Sep 1 11:35:42 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 1 Sep 2007 11:35:42 +0200 (CEST) Subject: [pypy-svn] r46233 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20070901093542.BA93981AD@code0.codespeak.net> Author: antocuni Date: Sat Sep 1 11:35:42 2007 New Revision: 46233 Modified: pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/test/test_runtest.py Log: implement ullong_mod. Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Sat Sep 1 11:35:42 2007 @@ -404,6 +404,7 @@ ootype.Signed:jvmgen.INTTOSTRINGI, ootype.Unsigned:jvmgen.PYPYSERIALIZEUINT, ootype.SignedLongLong:jvmgen.LONGTOSTRINGL, + ootype.UnsignedLongLong: jvmgen.PYPYSERIALIZEULONG, ootype.Float:jvmgen.DOUBLETOSTRINGD, ootype.Bool:jvmgen.PYPYSERIALIZEBOOLEAN, ootype.Void:jvmgen.PYPYSERIALIZEVOID, Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sat Sep 1 11:35:42 2007 @@ -394,6 +394,7 @@ PYPYUINTCMP = Method.s(jPyPy, 'uint_cmp', (jInt,jInt,), jInt) PYPYULONGCMP = Method.s(jPyPy, 'ulong_cmp', (jLong,jLong), jInt) PYPYUINTMOD = Method.s(jPyPy, 'uint_mod', (jInt, jInt), jInt) +PYPYULONGMOD = Method.s(jPyPy, 'ulong_mod', (jLong, jLong), jLong) PYPYUINTTODOUBLE = Method.s(jPyPy, 'uint_to_double', (jInt,), jDouble) PYPYDOUBLETOUINT = Method.s(jPyPy, 'double_to_uint', (jDouble,), jInt) PYPYDOUBLETOLONG = Method.s(jPyPy, 'double_to_long', (jDouble,), jLong) #PAUL @@ -410,6 +411,7 @@ PYPYDUMPEXCWRAPPER = Method.s(jPyPy, 'dump_exc_wrapper', (jObject,), jVoid) PYPYSERIALIZEBOOLEAN = Method.s(jPyPy, 'serialize_boolean', (jBool,), jString) PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) +PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,), jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Sat Sep 1 11:35:42 2007 @@ -210,8 +210,8 @@ 'ullong_div': jvmgen.LDIV, # valid? 'ullong_truediv': None, # TODO 'ullong_floordiv': jvmgen.LDIV, # valid? - 'ullong_mod': jvmgen.LREM, # valid? - 'ullong_mod_zer': _check_zer(jvmgen.LREM), + 'ullong_mod': jvmgen.PYPYULONGMOD, + 'ullong_mod_zer': _check_zer(jvmgen.PYPYULONGMOD), 'ullong_lt': 'ulong_less_than', 'ullong_le': 'ulong_less_equals', 'ullong_eq': 'ulong_equals', Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Sat Sep 1 11:35:42 2007 @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.Arrays; import java.util.Map; +import java.text.DecimalFormat; /** * Class with a number of utility routines. @@ -72,6 +73,12 @@ return (int)modulo; } + public static long ulong_mod(long x, long y) { + double dx = ulong_to_double(x); + double modulo = Math.IEEEremainder(dx, y); + return (long)modulo; + } + public static int ulong_cmp(long value1, long value2) { final int VALUE2BIGGER = -1; final int VALUE1BIGGER = 1; @@ -116,6 +123,17 @@ } } + public static double ulong_to_double(long value) { + if (value >= 0) + return value; + else { + long lopart = value & 0xFFFFFFFF; + long hipart = value >>> 32; + double result = (hipart << 32) | lopart; + return result; + } + } + public static int double_to_uint(double value) { if (value <= Integer.MAX_VALUE) return (int)value; @@ -249,6 +267,13 @@ } } + public static String serialize_ulonglong(long value) + { + double d = ulong_to_double(value); + DecimalFormat fmt = new DecimalFormat("0"); + return fmt.format(d); + } + public static String serialize_boolean(boolean l) { if (l) return "True"; Modified: pypy/dist/pypy/translator/jvm/test/test_runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_runtest.py Sat Sep 1 11:35:42 2007 @@ -1,6 +1,8 @@ +import sys from pypy.translator.jvm.test.runtest import JvmTest from pypy.translator.jvm.test.runtest import FLOAT_PRECISION from pypy.annotation.listdef import s_list_of_strings +from pypy.rlib.rarithmetic import r_ulonglong def ident(x): return x @@ -64,3 +66,6 @@ else: return None assert self.interpret(fn, [False]) is None + + def test_ullong(self): + assert self.interpret(ident, [r_ulonglong(sys.maxint+1)]) == sys.maxint+1 From antocuni at codespeak.net Sat Sep 1 14:28:48 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 1 Sep 2007 14:28:48 +0200 (CEST) Subject: [pypy-svn] r46234 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20070901122848.BFB348226@code0.codespeak.net> Author: antocuni Date: Sat Sep 1 14:28:46 2007 New Revision: 46234 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/test/test_op.py Log: correct implementation of uint_mul and uint_div Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sat Sep 1 14:28:46 2007 @@ -394,6 +394,8 @@ PYPYUINTCMP = Method.s(jPyPy, 'uint_cmp', (jInt,jInt,), jInt) PYPYULONGCMP = Method.s(jPyPy, 'ulong_cmp', (jLong,jLong), jInt) PYPYUINTMOD = Method.s(jPyPy, 'uint_mod', (jInt, jInt), jInt) +PYPYUINTMUL = Method.s(jPyPy, 'uint_mul', (jInt, jInt), jInt) +PYPYUINTDIV = Method.s(jPyPy, 'uint_div', (jInt, jInt), jInt) PYPYULONGMOD = Method.s(jPyPy, 'ulong_mod', (jLong, jLong), jLong) PYPYUINTTODOUBLE = Method.s(jPyPy, 'uint_to_double', (jInt,), jDouble) PYPYDOUBLETOUINT = Method.s(jPyPy, 'double_to_uint', (jDouble,), jInt) Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Sat Sep 1 14:28:46 2007 @@ -137,10 +137,10 @@ 'uint_add': jvmgen.IADD, 'uint_sub': jvmgen.ISUB, - 'uint_mul': jvmgen.IMUL, - 'uint_div': jvmgen.IDIV, # valid? - 'uint_truediv': None, # TODO - 'uint_floordiv': jvmgen.IDIV, # valid? + 'uint_mul': jvmgen.PYPYUINTMUL, +# 'uint_div': jvmgen.IDIV, # valid? +# 'uint_truediv': None, # TODO + 'uint_floordiv': jvmgen.PYPYUINTDIV, 'uint_mod': jvmgen.PYPYUINTMOD, 'uint_lt': 'u_less_than', 'uint_le': 'u_less_equals', Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Sat Sep 1 14:28:46 2007 @@ -73,6 +73,20 @@ return (int)modulo; } + public static int uint_mul(int x, int y) + { + long xx = uint_to_long(x); + long yy = uint_to_long(y); + return long_to_uint(xx * yy); + } + + public static int uint_div(int x, int y) + { + long xx = uint_to_long(x); + long yy = uint_to_long(y); + return long_to_uint(xx / yy); + } + public static long ulong_mod(long x, long y) { double dx = ulong_to_double(x); double modulo = Math.IEEEremainder(dx, y); @@ -113,14 +127,7 @@ public static final double BITS16 = (double)0xFFFF; public static double uint_to_double(int value) { - if (value >= 0) - return value; - else { - long loword = value & 0xFFFF; - long hiword = value >>> 16; - double result = (hiword << 16) | loword; - return result; - } + return (double)uint_to_long(value); } public static double ulong_to_double(long value) { @@ -137,13 +144,24 @@ public static int double_to_uint(double value) { if (value <= Integer.MAX_VALUE) return (int)value; + return long_to_uint((long)value); + } - long v = (long)value; - int loword = (int)(v & 0xFFFF); - int hiword = (int)(v >>> 16); + public static int long_to_uint(long value) + { + int loword = (int)(value & 0xFFFF); + int hiword = (int)(value >>> 16); return (hiword << 16) | loword; } + public static long uint_to_long(int value) + { + long loword = value & 0xFFFF; + long hiword = value >>> 16; + long res = (hiword << 16) | loword; + return res; + } + public static long double_to_long(double value) { //if (value <= LONG_MAX) @@ -259,12 +277,8 @@ public static String serialize_uint(int i) { if (i >= 0) return Integer.toString(i); - else { - long loword = i & 0xFFFF; - long hiword = i >>> 16; - long res = (hiword << 16) | loword; - return Long.toString(res); - } + else + return Long.toString(uint_to_long(i)); } public static String serialize_ulonglong(long value) Modified: pypy/dist/pypy/translator/jvm/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_op.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_op.py Sat Sep 1 14:28:46 2007 @@ -6,9 +6,6 @@ class TestOperations(JvmTest, BaseTestOperations): - def test_operations(self): - py.test.skip("Backend lacks appropriate precision") - def test_abs(self): py.test.skip("Backend lacks appropriate precision") From antocuni at codespeak.net Sat Sep 1 14:35:53 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 1 Sep 2007 14:35:53 +0200 (CEST) Subject: [pypy-svn] r46235 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20070901123553.0FBE6822A@code0.codespeak.net> Author: antocuni Date: Sat Sep 1 14:35:51 2007 New Revision: 46235 Modified: pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/test/test_op.py Log: all tests in test_op pass! Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Sat Sep 1 14:35:51 2007 @@ -7,10 +7,11 @@ from pypy.translator.oosupport.metavm import \ PushArg, PushAllArgs, StoreResult, InstructionList, New, DoNothing, Call,\ - SetField, GetField, DownCast, RuntimeNew, OOString, CastTo + SetField, GetField, DownCast, RuntimeNew, OOString, CastTo, PushPrimitive from pypy.translator.jvm.metavm import \ IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ CastPtrToWeakAddress, CastWeakAddressToPtr +from pypy.rpython.ootypesystem import ootype import pypy.translator.jvm.generator as jvmgen import pypy.translator.jvm.typesystem as jvmtype @@ -241,6 +242,7 @@ 'cast_float_to_uint': jvmgen.PYPYDOUBLETOUINT, 'truncate_longlong_to_int': jvmgen.L2I, 'cast_longlong_to_float': jvmgen.L2D, + 'is_early_constant': [PushPrimitive(ootype.Bool, False), StoreResult] } Modified: pypy/dist/pypy/translator/jvm/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_op.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_op.py Sat Sep 1 14:35:51 2007 @@ -5,13 +5,5 @@ # ====> ../../oosupport/test_template/operations.py class TestOperations(JvmTest, BaseTestOperations): - - def test_abs(self): - py.test.skip("Backend lacks appropriate precision") - - def test_is_true(self): - py.test.skip("VerifyError happens. Accessing uninit reg") - - def test_is_early_constant(self): - py.test.skip("Unknown opcode is_early_constant") + pass From arigo at codespeak.net Sat Sep 1 14:38:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Sep 2007 14:38:06 +0200 (CEST) Subject: [pypy-svn] r46236 - pypy/extradoc/talk/esug2007 Message-ID: <20070901123806.98355822D@code0.codespeak.net> Author: arigo Date: Sat Sep 1 14:38:03 2007 New Revision: 46236 Added: pypy/extradoc/talk/esug2007/ pypy/extradoc/talk/esug2007/Makefile - copied, changed from r45426, pypy/extradoc/talk/dyla2007/Makefile pypy/extradoc/talk/esug2007/abstract pypy/extradoc/talk/esug2007/beamerouterthememy.sty - copied unchanged from r45426, pypy/extradoc/talk/dyla2007/beamerouterthememy.sty pypy/extradoc/talk/esug2007/beamerthemeWarsaw.sty - copied unchanged from r45426, pypy/extradoc/talk/dyla2007/beamerthemeWarsaw.sty pypy/extradoc/talk/esug2007/talk.tex - copied, changed from r45426, pypy/extradoc/talk/dyla2007/talk.tex Log: The ESUG2007 talk, based on the Dyla'07 one - sorry, I forgot to update to the latest version - but anyway the talk was mostly only the demo, not the slides. Added: pypy/extradoc/talk/esug2007/abstract ============================================================================== --- (empty file) +++ pypy/extradoc/talk/esug2007/abstract Sat Sep 1 14:38:03 2007 @@ -0,0 +1,27 @@ + + +Description +----------- + +PyPy is a framework written in Python for generating virtual machines +for dynamic languages. The VMs are flexibly produced from a high-level +"specification" in the form of a simple interpreter for the dynamic +language. Low-level details like memory allocation and object layout +are not encoded manually, but inserted by the VM generation process. +This allows us to produce VMs that run within a wide range of execution +environments (from C-like to JVM/.NET). Interesting features can be +automatically inserted into the VMs at production time -- including a +good just-in-time compiler tuned to the dynamic language at hand. + +More than "selling" PyPy specifically, our aim in this project is to +raise interest in our approach to virtual machine construction. After a +demo and architecture overview, I will explain the context in which we +wrote PyPy, and how much of these ideas could be applied to a Smalltalk +context. + +Bio +--- + +/me has been an open source developer and post-doc working in Python on +language implementation issues for quite a few years now, after a +Ph.D. in mathematical logic at the Free University of Brussels. From arigo at codespeak.net Sun Sep 2 19:50:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Sep 2007 19:50:04 +0200 (CEST) Subject: [pypy-svn] r46243 - in pypy/branch/pypy-more-rtti-inprogress/rpython: lltypesystem module Message-ID: <20070902175004.D3C54818D@code0.codespeak.net> Author: arigo Date: Sun Sep 2 19:50:02 2007 New Revision: 46243 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Log: Start again with Windows porting. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py Sun Sep 2 19:50:02 2007 @@ -44,6 +44,7 @@ if max_n is not None and fieldname == S._arrayfld: cls = build_ctypes_array(FIELDTYPE, max_n) elif isinstance(FIELDTYPE, lltype.Ptr) and FIELDTYPE.TO is S: + # XXX this is not the right hack to handle circular structures! cls = ctypes.POINTER(S._name) else: cls = get_ctypes_type(FIELDTYPE) @@ -436,6 +437,9 @@ libraries = getattr(funcptr._obj, 'libraries', None) if not libraries: cfunc = getattr(standard_c_lib, funcname, None) + # XXX magic: on Windows try to load the function from 'kernel32' too + if cfunc is None and hasattr(ctypes, 'windll'): + cfunc = getattr(ctypes.windll.kernel32, funcname, None) else: cfunc = None for libname in libraries: Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Sun Sep 2 19:50:02 2007 @@ -30,12 +30,19 @@ register(function, [CLOCK_T], ...) """ - _includes_ = ['sys/times.h', 'utime.h', 'sys/types.h', 'unistd.h', - 'signal.h', 'wait.h'] + _includes_ = [] if not sys.platform.startswith('win'): - _includes_ += ['dirent.h', 'sys/stat.h'] - - CLOCK_T = platform.SimpleType('clock_t', rffi.INT) + _includes_ += ['dirent.h', 'sys/stat.h', + 'sys/times.h', 'utime.h', 'sys/types.h', 'unistd.h', + 'signal.h', 'wait.h'] + + CLOCK_T = platform.SimpleType('clock_t', rffi.INT) + + TMS = platform.Struct( + 'struct tms', [('tms_utime', rffi.INT), + ('tms_stime', rffi.INT), + ('tms_cutime', rffi.INT), + ('tms_cstime', rffi.INT)]) NAME_MAX = platform.DefinedConstantInteger('NAME_MAX') @@ -43,12 +50,6 @@ SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR') SEEK_END = platform.DefinedConstantInteger('SEEK_END') - TMS = platform.Struct( - 'struct tms', [('tms_utime', rffi.INT), - ('tms_stime', rffi.INT), - ('tms_cutime', rffi.INT), - ('tms_cstime', rffi.INT)]) - class RegisterOs(BaseLazyRegistering): _stringpolicy_ = 'fullauto' @@ -185,26 +186,64 @@ @registering(os.times) def register_os_times(self): - self.TMSP = lltype.Ptr(self.TMS) - os_times = self.llexternal('times', [self.TMSP], self.CLOCK_T) + if sys.platform.startswith('win'): + HANDLE = rffi.LONG + FILETIME = rffi.CStruct('_FILETIME', ('dwLowDateTime', rffi.LONG), + ('dwHighDateTime', rffi.LONG)) + GetCurrentProcess = self.llexternal('GetCurrentProcess', [], + HANDLE) + GetProcessTimes = self.llexternal('GetProcessTimes', + [HANDLE, + lltype.Ptr(FILETIME), + lltype.Ptr(FILETIME), + lltype.Ptr(FILETIME), + lltype.Ptr(FILETIME)], + lltype.Bool) + + def times_lltypeimpl(): + pcreate = lltype.malloc(FILETIME, flavor='raw') + pexit = lltype.malloc(FILETIME, flavor='raw') + pkernel = lltype.malloc(FILETIME, flavor='raw') + puser = lltype.malloc(FILETIME, flavor='raw') + hProc = GetCurrentProcess() + GetProcessTimes(hProc, pcreate, pexit, pkernel, puser) + # The fields of a FILETIME structure are the hi and lo parts + # of a 64-bit value expressed in 100 nanosecond units + # (of course). + result = (pkernel.c_dwHighDateTime*429.4967296 + + pkernel.c_dwLowDateTime*1E-7, + puser.c_dwHighDateTime*429.4967296 + + puser.c_dwLowDateTime*1E-7, + 0, 0, 0) + lltype.free(puser, flavor='raw') + lltype.free(pkernel, flavor='raw') + lltype.free(pexit, flavor='raw') + lltype.free(pcreate, flavor='raw') + return result + self.register(os.times, [], (float, float, float, float, float), + "ll_os.ll_times", llimpl=times_lltypeimpl) + return + + TMSP = lltype.Ptr(self.TMS) + os_times = self.llexternal('times', [TMSP], self.CLOCK_T) # Here is a random extra platform parameter which is important. # Strictly speaking, this should probably be retrieved at runtime, not # at translation time. - self.CLOCK_TICKS_PER_SECOND = float(os.sysconf('SC_CLK_TCK')) + CLOCK_TICKS_PER_SECOND = float(os.sysconf('SC_CLK_TCK')) def times_lltypeimpl(): - l_tmsbuf = lltype.malloc(self.TMSP.TO, flavor='raw') + l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw') try: result = os_times(l_tmsbuf) if result == -1: raise OSError(rffi.get_errno(), "times failed") return ( - l_tmsbuf.c_tms_utime / self.CLOCK_TICKS_PER_SECOND, - l_tmsbuf.c_tms_stime / self.CLOCK_TICKS_PER_SECOND, - l_tmsbuf.c_tms_cutime / self.CLOCK_TICKS_PER_SECOND, - l_tmsbuf.c_tms_cstime / self.CLOCK_TICKS_PER_SECOND, - result / self.CLOCK_TICKS_PER_SECOND) + l_tmsbuf.c_tms_utime / CLOCK_TICKS_PER_SECOND, + l_tmsbuf.c_tms_stime / CLOCK_TICKS_PER_SECOND, + l_tmsbuf.c_tms_cutime / CLOCK_TICKS_PER_SECOND, + l_tmsbuf.c_tms_cstime / CLOCK_TICKS_PER_SECOND, + result / CLOCK_TICKS_PER_SECOND) finally: lltype.free(l_tmsbuf, flavor='raw') self.register(os.times, [], (float, float, float, float, float), @@ -264,11 +303,11 @@ return extdef([], (str, str, str, str, str), "ll_os.ll_uname", llimpl=uname_llimpl) - @registering(os.getuid) + @registering_if(os, 'getuid') def register_os_getuid(self): return self.extdef_for_os_function_returning_int('getuid') - @registering(os.geteuid) + @registering_if(os, 'geteuid') def register_os_geteuid(self): return self.extdef_for_os_function_returning_int('geteuid') @@ -428,7 +467,12 @@ @registering(os.access) def register_os_access(self): - os_access = self.llexternal('access', [rffi.CCHARP, rffi.INT], + if sys.platform.startswith('win'): + name = '_access' + else: + name = 'access' + os_access = self.llexternal(name, + [rffi.CCHARP, rffi.INT], rffi.INT) def access_llimpl(path, mode): From arigo at codespeak.net Sun Sep 2 19:56:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Sep 2007 19:56:10 +0200 (CEST) Subject: [pypy-svn] r46244 - in pypy/branch/pypy-more-rtti-inprogress/rpython/module: . test Message-ID: <20070902175610.061798187@code0.codespeak.net> Author: arigo Date: Sun Sep 2 19:56:09 2007 New Revision: 46244 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_ll_os.py Log: test_ll_os.py passes on Windows. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Sun Sep 2 19:56:09 2007 @@ -50,6 +50,11 @@ SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR') SEEK_END = platform.DefinedConstantInteger('SEEK_END') +if sys.platform.startswith('win'): + underscore_on_windows = '_' +else: + underscore_on_windows = '' + class RegisterOs(BaseLazyRegistering): _stringpolicy_ = 'fullauto' @@ -467,11 +472,7 @@ @registering(os.access) def register_os_access(self): - if sys.platform.startswith('win'): - name = '_access' - else: - name = 'access' - os_access = self.llexternal(name, + os_access = self.llexternal(underscore_on_windows + 'access', [rffi.CCHARP, rffi.INT], rffi.INT) @@ -488,7 +489,7 @@ @registering(os.getcwd) def register_os_getcwd(self): - os_getcwd = self.llexternal('getcwd', + os_getcwd = self.llexternal(underscore_on_windows + 'getcwd', [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP, stringpolicy='noauto') Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_ll_os.py Sun Sep 2 19:56:09 2007 @@ -51,6 +51,8 @@ def test_os_wstar(): from pypy.rpython.module.ll_os import RegisterOs for name in RegisterOs.w_star: + if not hasattr(os, name): + continue def fun(s): return getattr(os, name)(s) From fijal at codespeak.net Sun Sep 2 20:03:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 2 Sep 2007 20:03:49 +0200 (CEST) Subject: [pypy-svn] r46245 - in pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem: . test Message-ID: <20070902180349.D23028187@code0.codespeak.net> Author: fijal Date: Sun Sep 2 20:03:49 2007 New Revision: 46245 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/test/test_ll2ctypes.py Log: Remove this hack and skip the test Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py Sun Sep 2 20:03:49 2007 @@ -43,9 +43,6 @@ FIELDTYPE = S._flds[fieldname] if max_n is not None and fieldname == S._arrayfld: cls = build_ctypes_array(FIELDTYPE, max_n) - elif isinstance(FIELDTYPE, lltype.Ptr) and FIELDTYPE.TO is S: - # XXX this is not the right hack to handle circular structures! - cls = ctypes.POINTER(S._name) else: cls = get_ctypes_type(FIELDTYPE) fields.append((fieldname, cls)) Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/test/test_ll2ctypes.py Sun Sep 2 20:03:49 2007 @@ -406,7 +406,7 @@ lltype.free(s, flavor='raw') def test_recursive_struct(self): - # XXX sucky way of creating those + py.test.skip("Not implemented") SX = lltype.ForwardReference() S1 = lltype.Struct('S1', ('x', lltype.Ptr(SX))) S1.x.TO.become(S1) From santagada at codespeak.net Sun Sep 2 23:53:13 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 2 Sep 2007 23:53:13 +0200 (CEST) Subject: [pypy-svn] r46247 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070902215313.1995381FD@code0.codespeak.net> Author: santagada Date: Sun Sep 2 23:53:12 2007 New Revision: 46247 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Log: in the wait manpage for os x and freebsd this is the right place for the signal header. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Sun Sep 2 23:53:12 2007 @@ -34,7 +34,7 @@ if not sys.platform.startswith('win'): _includes_ += ['dirent.h', 'sys/stat.h', 'sys/times.h', 'utime.h', 'sys/types.h', 'unistd.h', - 'signal.h', 'wait.h'] + 'signal.h', 'sys/wait.h'] CLOCK_T = platform.SimpleType('clock_t', rffi.INT) From santagada at codespeak.net Sun Sep 2 23:59:19 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 2 Sep 2007 23:59:19 +0200 (CEST) Subject: [pypy-svn] r46248 - pypy/dist/pypy/doc Message-ID: <20070902215919.4E26A81FD@code0.codespeak.net> Author: santagada Date: Sun Sep 2 23:59:18 2007 New Revision: 46248 Modified: pypy/dist/pypy/doc/faq.txt Log: little doc correction, put the ctypes version needed and saying that cpython 2.5 is supported now Modified: pypy/dist/pypy/doc/faq.txt ============================================================================== --- pypy/dist/pypy/doc/faq.txt (original) +++ pypy/dist/pypy/doc/faq.txt Sun Sep 2 23:59:18 2007 @@ -51,8 +51,8 @@ OS X and mostly works under Windows too (but is tested there less extensively). PyPy needs a CPython running on the target platform to bootstrap, as cross compilation is not really meant to work yet. -At the moment you need CPython 2.4 for the translation process, 2.5 is -not fully supported. +At the moment you need CPython 2.4 (with ctypes 0.9.9.6 or newer) or +CPython 2.5 for the translation process. Currently (due to time restrictions) we are not trying hard to support PyPy in a 64 bit environment. While things seem to mostly work, a few From arigo at codespeak.net Mon Sep 3 11:00:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 11:00:24 +0200 (CEST) Subject: [pypy-svn] r46257 - in pypy/branch/pypy-more-rtti-inprogress/rpython: . module Message-ID: <20070903090024.A7452815C@code0.codespeak.net> Author: arigo Date: Mon Sep 3 11:00:24 2007 New Revision: 46257 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/extfunc.py pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Log: More Windozification... Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/extfunc.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/extfunc.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/extfunc.py Mon Sep 3 11:00:24 2007 @@ -95,6 +95,9 @@ __ATTRIBUTES = ['includes', 'include_dirs'] def configure(self, CConfig): + classes_seen = self.__dict__.setdefault('__classes_seen', {}) + if CConfig in classes_seen: + return from pypy.rpython.tool import rffi_platform as platform # copy some stuff for item in self.__ATTRIBUTES: @@ -102,6 +105,7 @@ if value: setattr(self, '_%s_' % item, value) self.__dict__.update(platform.configure(CConfig)) + classes_seen[CConfig] = True def llexternal(self, *args, **kwds): kwds = kwds.copy() @@ -117,6 +121,9 @@ kwds['stringpolicy'] = stringpolicy return rffi.llexternal(*args, **kwds) + def _freeze_(self): + return True + class genericcallable(object): """ A way to specify the callable annotation, but deferred until we have bookkeeper Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Mon Sep 3 11:00:24 2007 @@ -19,12 +19,14 @@ CLOCK_T = platform.SimpleType('clock_t', rffi.INT) TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.INT), ('tv_usec', rffi.INT)]) - TIMEB = platform.Struct('struct timeb', [('time', rffi.INT), - ('millitm', rffi.INT)]) TIME_T = platform.SimpleType('time_t', rffi.INT) HAVE_GETTIMEOFDAY = platform.Has('gettimeofday') HAVE_FTIME = platform.Has('ftime') +class CConfigForFTime: + TIMEB = platform.Struct('struct timeb', [('time', rffi.INT), + ('millitm', rffi.INT)]) + constant_names = ['CLOCKS_PER_SEC', 'CLK_TCK', 'EINTR'] for const in constant_names: setattr(CConfig, const, platform.DefinedConstantInteger(const)) @@ -41,7 +43,6 @@ else: self.CLOCKS_PER_SEC = self.CLK_TCK self.TIMEVALP = lltype.Ptr(self.TIMEVAL) - self.TIMEBP = lltype.Ptr(self.TIMEB) @registering(time.time) def register_time_time(self): @@ -60,9 +61,9 @@ c_gettimeofday = None if self.HAVE_FTIME: - c_ftime = self.llexternal('ftime', [self.TIMEBP], lltype.Void) - else: - c_ftime = None # not to confuse flow objspace + self.configure(CConfigForFTime) + c_ftime = self.llexternal('ftime', [lltype.Ptr(self.TIMEB)], + lltype.Void) c_time = self.llexternal('time', [rffi.VOIDP], self.TIME_T) From arigo at codespeak.net Mon Sep 3 11:13:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 11:13:19 +0200 (CEST) Subject: [pypy-svn] r46258 - in pypy/branch/pypy-more-rtti-inprogress: objspace/flow rpython/module Message-ID: <20070903091319.06E258153@code0.codespeak.net> Author: arigo Date: Mon Sep 3 11:13:18 2007 New Revision: 46258 Modified: pypy/branch/pypy-more-rtti-inprogress/objspace/flow/objspace.py pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Log: time.time() -> win Modified: pypy/branch/pypy-more-rtti-inprogress/objspace/flow/objspace.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/objspace/flow/objspace.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/objspace/flow/objspace.py Mon Sep 3 11:13:18 2007 @@ -241,7 +241,9 @@ if func.func_closure is None: closure = None else: - closure = [extract_cell_content(c) for c in func.func_closure] + closure = [extract_cell_content(c, name, func) + for c, name in zip(func.func_closure, + func.func_code.co_freevars)] # CallableFactory.pycall may add class_ to functions that are methods name = func.func_name class_ = getattr(func, 'class_', None) @@ -558,7 +560,7 @@ OverflowError) # for the float case del _add_exceptions, _add_except_ovf -def extract_cell_content(c): +def extract_cell_content(c, varname, func): """Get the value contained in a CPython 'cell', as read through the func_closure of a function object.""" # yuk! this is all I could come up with that works in Python 2.2 too @@ -572,7 +574,11 @@ x = X() x_cell, = (lambda: x).func_closure x_cell == c - return x.other # crashes if the cell is actually empty + try: + return x.other # crashes if the cell is actually empty + except AttributeError: + raise Exception("in %r, the free variable %r has no value" % ( + func, varname)) def make_op(name, symbol, arity, specialnames): if hasattr(FlowObjSpace, name): Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Mon Sep 3 11:13:18 2007 @@ -46,51 +46,50 @@ @registering(time.time) def register_time_time(self): - if sys.platform == 'win32': - xxx + # AWFUL + if self.HAVE_GETTIMEOFDAY: + if self.GETTIMEOFDAY_NO_TZ: + c_gettimeofday = self.llexternal('gettimeofday', + [self.TIMEVALP], rffi.INT) + else: + c_gettimeofday = self.llexternal('gettimeofday', + [self.TIMEVALP, rffi.VOIDP], rffi.INT) + else: + c_gettimeofday = None + + if self.HAVE_FTIME: + self.configure(CConfigForFTime) + c_ftime = self.llexternal('ftime', [lltype.Ptr(self.TIMEB)], + lltype.Void) else: - # AWFUL + c_ftime = None # to not confuse the flow space + + c_time = self.llexternal('time', [rffi.VOIDP], self.TIME_T) + + def time_time_llimpl(): + void = lltype.nullptr(rffi.VOIDP.TO) + result = -1.0 if self.HAVE_GETTIMEOFDAY: + t = lltype.malloc(self.TIMEVAL, flavor='raw') + if self.GETTIMEOFDAY_NO_TZ: - c_gettimeofday = self.llexternal('gettimeofday', - [self.TIMEVALP], rffi.INT) + if c_gettimeofday(t) == 0: + result = float(t.c_tv_sec) + \ + float(t.c_tv_usec) * 0.000001 else: - c_gettimeofday = self.llexternal('gettimeofday', - [self.TIMEVALP, rffi.VOIDP], rffi.INT) - else: - c_gettimeofday = None - + if c_gettimeofday(t, void) == 0: + result = float(t.c_tv_sec) + \ + float(t.c_tv_usec) * 0.000001 + lltype.free(t, flavor='raw') + if result != -1: + return result if self.HAVE_FTIME: - self.configure(CConfigForFTime) - c_ftime = self.llexternal('ftime', [lltype.Ptr(self.TIMEB)], - lltype.Void) - - c_time = self.llexternal('time', [rffi.VOIDP], self.TIME_T) - - def time_time_llimpl(): - void = lltype.nullptr(rffi.VOIDP.TO) - result = -1.0 - if self.HAVE_GETTIMEOFDAY: - t = lltype.malloc(self.TIMEVAL, flavor='raw') - - if self.GETTIMEOFDAY_NO_TZ: - if c_gettimeofday(t) == 0: - result = float(t.c_tv_sec) + \ - float(t.c_tv_usec) * 0.000001 - else: - if c_gettimeofday(t, void) == 0: - result = float(t.c_tv_sec) + \ - float(t.c_tv_usec) * 0.000001 - lltype.free(t, flavor='raw') - if result != -1: - return result - if self.HAVE_FTIME: - t = lltype.malloc(self.TIMEB, flavor='raw') - c_ftime(t) - result = float(int(t.c_time)) + float(int(t.c_millitm)) * 0.001 - lltype.free(t, flavor='raw') - return result - return float(c_time(void)) + t = lltype.malloc(self.TIMEB, flavor='raw') + c_ftime(t) + result = float(int(t.c_time)) + float(int(t.c_millitm)) * 0.001 + lltype.free(t, flavor='raw') + return result + return float(c_time(void)) return extdef([], float, llimpl=time_time_llimpl, export_name='ll_time.ll_time_time') From arigo at codespeak.net Mon Sep 3 11:33:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 11:33:27 +0200 (CEST) Subject: [pypy-svn] r46259 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903093327.DBAC48153@code0.codespeak.net> Author: arigo Date: Mon Sep 3 11:33:27 2007 New Revision: 46259 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Log: finish ll_time -> win. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Mon Sep 3 11:33:27 2007 @@ -97,9 +97,28 @@ @registering(time.clock) def register_time_clock(self): c_clock = self.llexternal('clock', [], self.CLOCK_T) - if sys.platform == 'win32': - xxx + # hacking to avoid LARGE_INTEGER which is a union... + A = lltype.FixedSizeArray(lltype.SignedLongLong, 1) + QueryPerformanceCounter = self.llexternal( + 'QueryPerformanceCounter', [lltype.Ptr(A)], lltype.Void) + QueryPerformanceFrequency = self.llexternal( + 'QueryPerformanceFrequency', [lltype.Ptr(A)], rffi.INT) + class State(object): + pass + state = State() + state.divisor = 0.0 + def time_clock_llimpl(): + a = lltype.malloc(A, flavor='raw') + if state.divisor == 0.0: + QueryPerformanceCounter(a) + state.counter_start = a[0] + QueryPerformanceFrequency(a) + state.divisor = float(a[0]) + QueryPerformanceCounter(a) + diff = a[0] - state.counter_start + lltype.free(a, flavor='raw') + return float(diff) / state.divisor else: def time_clock_llimpl(): result = c_clock() @@ -111,15 +130,20 @@ @registering(time.sleep) def register_time_sleep(self): if sys.platform == 'win32': - xxx + MAX = sys.maxint + Sleep = self.llexternal('Sleep', [rffi.ULONG], lltype.Void) + def time_sleep_llimpl(secs): + millisecs = secs * 1000.0 + while millisecs > MAX: + Sleep(MAX) + millisecs -= MAX + Sleep(rffi.cast(rffi.ULONG, int(millisecs))) else: c_select = self.llexternal('select', [rffi.INT, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, self.TIMEVALP], rffi.INT) def time_sleep_llimpl(secs): - # XXX cannot put it into try: finally: because - # someone forgotten to call hop.exception_is_here void = lltype.nullptr(rffi.VOIDP.TO) t = lltype.malloc(self.TIMEVAL, flavor='raw') try: From antocuni at codespeak.net Mon Sep 3 11:35:14 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 11:35:14 +0200 (CEST) Subject: [pypy-svn] r46260 - pypy/dist/pypy/translator/jvm Message-ID: <20070903093514.90BF38153@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 11:35:13 2007 New Revision: 46260 Modified: pypy/dist/pypy/translator/jvm/genjvm.py Log: recompile *.java files only when needed. Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Mon Sep 3 11:35:13 2007 @@ -108,21 +108,27 @@ def _compile_helper(self, clsnms): # HACK: compile the Java helper class. Should eventually # use rte.py - tocompile = [] - for clsnm in clsnms: - pypycls = self.classdir.join(clsnm + '.class') - if not pypycls.check(): - tocompile.append(clsnm) - if tocompile: - thisdir = py.magic.autopath().dirpath() - javasrcs = [str(thisdir.join('src/pypy', clsnm + '.java')) for - clsnm in tocompile] - self._invoke([getoption('javac'), - '-nowarn', - '-d', str(self.classdir)]+ - javasrcs, - True) - + thisdir = py.magic.autopath().dirpath() + rootdir = thisdir.join('src') + srcdir = rootdir.join('pypy') + javafiles = srcdir.listdir('*.java') + classfiles = srcdir.listdir('*.class') + + recompile = True + if classfiles: + last_modified_java = max([java.mtime() for java in javafiles]) + last_modified_class = max([cls.mtime() for cls in classfiles]) + if last_modified_java < last_modified_class: + recompile = False + + if recompile: + log.red('Compiling java classes') + javasrcs = [str(srcdir.join(clsnm + '.java')) for clsnm in clsnms] + self._invoke([getoption('javac'), '-nowarn', '-d', str(rootdir)] + javasrcs, True) + + # copy .class files to classdir + for classfile in srcdir.listdir('*.class'): + classfile.copy(self.classdir.join('pypy')) def compile(self): """ From antocuni at codespeak.net Mon Sep 3 11:37:05 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 11:37:05 +0200 (CEST) Subject: [pypy-svn] r46261 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070903093705.DE8CA815C@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 11:37:04 2007 New Revision: 46261 Modified: pypy/dist/pypy/translator/jvm/test/test_cast.py Log: test_uint_to_float passes Modified: pypy/dist/pypy/translator/jvm/test/test_cast.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_cast.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_cast.py Mon Sep 3 11:37:04 2007 @@ -3,7 +3,4 @@ from pypy.translator.oosupport.test_template.cast import BaseTestCast class TestCast(BaseTestCast, JvmTest): - - def test_uint_to_float(self): - # This is most likely with how we render uints when we print them, and they get parsed. - py.test.skip('Same issue seen in other tests with uints... 2147450880.0 == 2147483648.0') + pass From arigo at codespeak.net Mon Sep 3 12:23:36 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 12:23:36 +0200 (CEST) Subject: [pypy-svn] r46263 - in pypy/branch/pypy-more-rtti-inprogress: rpython/module translator/c/test Message-ID: <20070903102336.8A5F7816A@code0.codespeak.net> Author: arigo Date: Mon Sep 3 12:23:36 2007 New Revision: 46263 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_stat.py pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py Log: os.stat() returns bogus times on Windows. Skip the tests for now. This check-in contains the start of the "correct" implementation and the reason why I stopped working on it for now. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_stat.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_stat.py Mon Sep 3 12:23:36 2007 @@ -199,3 +199,58 @@ "ll_os.ll_os_%s" % (name,), llimpl=func_with_new_name(os_mystat_llimpl, 'os_%s_llimpl' % (name,))) + +# ____________________________________________________________ + +if 0: + XXX - """ + disabled for now: + error codes are different when returned from the Win32 API, + which makes things a mess that I don't want to tackle now... + """ + # The CRT of Windows has a number of flaws wrt. its stat() implementation: + # - for when we implement subsecond resolution in RPython, time stamps + # would be restricted to second resolution + # - file modification times suffer from forth-and-back conversions between + # UTC and local time + # Therefore, we implement our own stat, based on the Win32 API directly. + from pypy.rpython.tool import rffi_platform as platform + + assert len(STAT_FIELDS) == 10 # no extra fields on Windows + FILETIME = rffi.CStruct('_FILETIME', ('dwLowDateTime', rffi.LONG), + ('dwHighDateTime', rffi.LONG)) + class CConfig: + GET_FILEEX_INFO_LEVELS = platform.SimpleType('GET_FILEEX_INFO_LEVELS', + rffi.INT) + GetFileExInfoStandard = platform.ConstantInteger( + 'GetFileExInfoStandard') + WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( + '_WIN32_FILE_ATTRIBUTE_DATA', + [('dwFileAttributes', rffi.ULONG), + ('nFileSizeHigh', rffi.ULONG), + ('nFileSizeLow', rffi.ULONG), + ('ftCreationTime', FILETIME), + ('ftLastAccessTime', FILETIME), + ('ftCreationTime', FILETIME)]) + + globals().update(platform.configure(CConfig)) + + GetFileAttributesEx = rffi.llexternal( + 'GetFileAttributesExA', [rffi.CCHARP, + GET_FILEEX_INFO_LEVELS, + lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], + rffi.INT) + + def os_stat_llimpl(path): + data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + try: + l_path = rffi.str2charp(path) + res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + rffi.free_charp(l_path) + if res == 0: + # ignore the GetLastError() which is a number that we cannot + # easily report... + XXX + YYY + finally: + lltype.free(data, flavor='raw') Modified: pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py Mon Sep 3 12:23:36 2007 @@ -176,6 +176,8 @@ res = f() assert res[0] == os.stat(filename).st_mode assert res[1] == os.stat(filename).st_ino + if sys.platform.startswith('win'): + py.test.skip("in-progress - bogus stat().st_time") st_ctime = res[2] if isinstance(st_ctime, float): assert st_ctime == os.stat(filename).st_ctime @@ -200,6 +202,8 @@ os.close(fd) assert st_mode == osstat.st_mode assert st_ino == osstat.st_ino + if sys.platform.startswith('win'): + py.test.skip("in-progress - bogus stat().st_time") if isinstance(st_mtime, float): assert st_mtime == osstat.st_mtime else: From arigo at codespeak.net Mon Sep 3 12:38:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 12:38:56 +0200 (CEST) Subject: [pypy-svn] r46266 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903103856.9828A814F@code0.codespeak.net> Author: arigo Date: Mon Sep 3 12:38:56 2007 New Revision: 46266 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Log: Make the RPython interface to os.mkdir() always take a mode argument, which we just ignore on Windows (CPython's os.mkdir() behaves like that too). Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Mon Sep 3 12:38:56 2007 @@ -716,21 +716,21 @@ @registering(os.mkdir) def register_os_mkdir(self): if os.name == 'nt': - ARG2 = [] # no 'mode' argument on Windows - arg2 = [] + ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - arg2 = [int] os_mkdir = self.llexternal('mkdir', [rffi.CCHARP]+ARG2, rffi.INT) + IGNORE_MODE = len(ARG2) == 0 - def mkdir_llimpl(pathname, *opt_mode): - if opt_mode: - opt_mode = (opt_mode[0],) - res = os_mkdir(pathname, *opt_mode) + def mkdir_llimpl(pathname, mode): + if IGNORE_MODE: + res = os_mkdir(pathname) + else: + res = os_mkdir(pathname, mode) if res < 0: raise OSError(rffi.get_errno(), "os_mkdir failed") - return extdef([str]+arg2, s_None, llimpl=mkdir_llimpl, + return extdef([str, int], s_None, llimpl=mkdir_llimpl, export_name="ll_os.ll_os_mkdir") @registering(os.rmdir) From arigo at codespeak.net Mon Sep 3 13:40:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 13:40:27 +0200 (CEST) Subject: [pypy-svn] r46268 - in pypy/branch/pypy-more-rtti-inprogress/rpython: lltypesystem module Message-ID: <20070903114027.1475380C0@code0.codespeak.net> Author: arigo Date: Mon Sep 3 13:40:26 2007 New Revision: 46268 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Log: Windozify listdir() and pipe(). Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py Mon Sep 3 13:40:26 2007 @@ -96,6 +96,9 @@ if os.name != 'nt': TYPES.append('mode_t') TYPES.append('pid_t') +else: + MODE_T = lltype.Signed + PID_T = lltype.Signed def setup(): """ creates necessary c-level types Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Mon Sep 3 13:40:26 2007 @@ -44,8 +44,6 @@ ('tms_cutime', rffi.INT), ('tms_cstime', rffi.INT)]) - NAME_MAX = platform.DefinedConstantInteger('NAME_MAX') - SEEK_SET = platform.DefinedConstantInteger('SEEK_SET') SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR') SEEK_END = platform.DefinedConstantInteger('SEEK_END') @@ -192,7 +190,7 @@ @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): - HANDLE = rffi.LONG + HANDLE = rffi.ULONG FILETIME = rffi.CStruct('_FILETIME', ('dwLowDateTime', rffi.LONG), ('dwHighDateTime', rffi.LONG)) GetCurrentProcess = self.llexternal('GetCurrentProcess', [], @@ -322,11 +320,7 @@ @registering(os.open) def register_os_open(self): - if os.name == 'nt': - mode_t = rffi.INT - else: - mode_t = rffi.MODE_T - os_open = self.llexternal('open', [rffi.CCHARP, rffi.INT, mode_t], + os_open = self.llexternal('open', [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) def os_open_llimpl(path, flags, mode): @@ -523,14 +517,80 @@ def register_os_listdir(self): # we need a different approach on Windows and on Posix if sys.platform.startswith('win'): - XXX # FindFirstFile, FindNextFile - else: + class CConfig: + _includes_ = ['windows.h'] + WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', + [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) + INVALID_HANDLE_VALUE = platform.ConstantInteger( + 'INVALID_HANDLE_VALUE') + ERROR_FILE_NOT_FOUND = platform.ConstantInteger( + 'ERROR_FILE_NOT_FOUND') + ERROR_NO_MORE_FILES = platform.ConstantInteger( + 'ERROR_NO_MORE_FILES') + + config = platform.configure(CConfig) + WIN32_FIND_DATA = config['WIN32_FIND_DATA'] + INVALID_HANDLE_VALUE = config['INVALID_HANDLE_VALUE'] + ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] + ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] + LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) + HANDLE = rffi.ULONG + #MAX_PATH = WIN32_FIND_DATA.c_cFileName.length + + GetLastError = self.llexternal('GetLastError', [], lltype.Signed) + FindFirstFile = self.llexternal('FindFirstFile', + [rffi.CCHARP, LPWIN32_FIND_DATA], + HANDLE) + FindNextFile = self.llexternal('FindNextFile', + [HANDLE, LPWIN32_FIND_DATA], + rffi.INT) + FindClose = self.llexternal('FindClose', + [HANDLE], + rffi.INT) + + def os_listdir_llimpl(path): + if path and path[-1] not in ('/', '\\', ':'): + path += '/' + path += '*.*' + filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') + try: + result = [] + hFindFile = FindFirstFile(path, filedata) + if hFindFile == INVALID_HANDLE_VALUE: + error = GetLastError() + if error == ERROR_FILE_NOT_FOUND: + return result + else: + # XXX guess error code :-( + raise OSError(errno.ENOENT, "FindFirstFile failed") + while True: + name = rffi.charp2str(rffi.cast(rffi.CCHARP, + filedata.c_cFileName)) + if name != "." and name != "..": # skip these + result.append(name) + if not FindNextFile(hFindFile, filedata): + break + # FindNextFile sets error to ERROR_NO_MORE_FILES if + # it got to the end of the directory + error = GetLastError() + FindClose(hFindFile) + if error == ERROR_NO_MORE_FILES: + return result + else: + # XXX guess error code :-( + raise OSError(errno.EIO, "FindNextFile failed") + finally: + lltype.free(filedata, flavor='raw') + + else: + class CConfig: + DIRENT = platform.Struct('dirent', + [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]) + + config = platform.configure(CConfig) + DIRENT = config['DIRENT'] + DIRENTP = lltype.Ptr(DIRENT) DIRP = rffi.COpaquePtr('DIR') - DIRENTP = rffi.CStructPtr('dirent', - ('d_name', lltype.FixedSizeArray(lltype.Char, self.NAME_MAX+1)), - ) - # XXX so far, DIRENTP cannot be handled by ll2ctypes because - # it contains other fields before 'd_name', at least on Linux os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP) os_readdir = self.llexternal('readdir', [DIRP], DIRENTP) os_closedir = self.llexternal('closedir', [DIRP], rffi.INT) @@ -555,16 +615,41 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", - llimpl=os_listdir_llimpl) + return extdef([str], # a single argument which is a str + [str], # returns a list of strings + "ll_os.ll_os_listdir", + llimpl=os_listdir_llimpl) @registering(os.pipe) def register_os_pipe(self): # we need a different approach on Windows and on Posix if sys.platform.startswith('win'): - XXX # CreatePipe, _open_osfhandle + HANDLE = rffi.ULONG + HANDLEP = lltype.Ptr(lltype.FixedSizeArray(HANDLE, 1)) + CreatePipe = self.llexternal('CreatePipe', [HANDLEP, + HANDLEP, + rffi.VOIDP, + rffi.ULONG], + rffi.INT) + _open_osfhandle = self.llexternal('_open_osfhandle', [rffi.ULONG, + rffi.INT], + rffi.INT) + null = lltype.nullptr(rffi.VOIDP.TO) + + def os_pipe_llimpl(): + pread = lltype.malloc(HANDLEP.TO, flavor='raw') + pwrite = lltype.malloc(HANDLEP.TO, flavor='raw') + ok = CreatePipe(pread, pwrite, null, 0) + hread = pread[0] + hwrite = pwrite[0] + lltype.free(pwrite, flavor='raw') + lltype.free(pread, flavor='raw') + if not ok: # XXX guess the error, can't use GetLastError() + raise OSError(errno.EMFILE, "os_pipe failed") + fdread = _open_osfhandle(hread, 0) + fdwrite = _open_osfhandle(hwrite, 1) + return (fdread, fdwrite) + else: INT_ARRAY_P = lltype.Ptr(lltype.FixedSizeArray(rffi.INT, 2)) os_pipe = self.llexternal('pipe', [INT_ARRAY_P], rffi.INT) @@ -579,9 +664,9 @@ raise OSError(rffi.get_errno(), "os_pipe failed") return (read_fd, write_fd) - return extdef([], (int, int), - "ll_os.ll_os_pipe", - llimpl=os_pipe_llimpl) + return extdef([], (int, int), + "ll_os.ll_os_pipe", + llimpl=os_pipe_llimpl) @registering_if(os, 'readlink') def register_os_readlink(self): @@ -747,14 +832,11 @@ @registering(os.chmod) def register_os_chmod(self): - if os.name == 'nt': - MODE_T = rffi.INT - else: - MODE_T = rffi.MODE_T - os_chmod = self.llexternal('chmod', [rffi.CCHARP, MODE_T], rffi.INT) + os_chmod = self.llexternal('chmod', [rffi.CCHARP, rffi.MODE_T], + rffi.INT) def chmod_llimpl(path, mode): - res = os_chmod(path, rffi.cast(MODE_T, mode)) + res = os_chmod(path, rffi.cast(rffi.MODE_T, mode)) if res < 0: raise OSError(rffi.get_errno(), "os_chmod failed") @@ -776,14 +858,10 @@ @registering(os.umask) def register_os_umask(self): - if os.name == 'nt': - mode_t = rffi.INT - else: - mode_t = rffi.MODE_T - os_umask = self.llexternal('umask', [mode_t], mode_t) + os_umask = self.llexternal('umask', [rffi.MODE_T], rffi.MODE_T) def umask_llimpl(fd): - res = os_umask(rffi.cast(mode_t, fd)) + res = os_umask(rffi.cast(rffi.MODE_T, fd)) return rffi.cast(lltype.Signed, res) return extdef([int], int, llimpl=umask_llimpl, From arigo at codespeak.net Mon Sep 3 13:49:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 13:49:21 +0200 (CEST) Subject: [pypy-svn] r46270 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903114921.37C7280E9@code0.codespeak.net> Author: arigo Date: Mon Sep 3 13:49:19 2007 New Revision: 46270 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Log: Posix fix. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_time.py Mon Sep 3 13:49:19 2007 @@ -24,6 +24,7 @@ HAVE_FTIME = platform.Has('ftime') class CConfigForFTime: + _includes_ = ['sys/timeb.h'] TIMEB = platform.Struct('struct timeb', [('time', rffi.INT), ('millitm', rffi.INT)]) From arigo at codespeak.net Mon Sep 3 14:11:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 14:11:59 +0200 (CEST) Subject: [pypy-svn] r46273 - pypy/branch/pypy-more-rtti-inprogress/rpython/test Message-ID: <20070903121159.7E8878168@code0.codespeak.net> Author: arigo Date: Mon Sep 3 14:11:56 2007 New Revision: 46273 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_llann.py Log: A test showing an annotation crash. Annoying. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_llann.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_llann.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_llann.py Mon Sep 3 14:11:56 2007 @@ -315,7 +315,25 @@ s = self.annotate(llf, [annmodel.SomeInteger()]) assert s.unsigned == True - + def test_pbctype(self): + py.test.skip("annotation crash") + TYPE = Void + TYPE2 = Signed + def g(lst): + n = lst[0] + if isinstance(TYPE, Number): + result = 123 + else: + result = 456 + if isinstance(TYPE2, Number): + result += 1 + return result + n + def llf(): + lst = [5] + g(lst) + lst.append(6) + self.annotate(llf, []) + def test_pseudohighlevelcallable(): t = TranslationContext() t.buildannotator() From arigo at codespeak.net Mon Sep 3 14:27:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 14:27:47 +0200 (CEST) Subject: [pypy-svn] r46274 - in pypy/branch/pypy-more-rtti-inprogress: rpython/module translator/c/test Message-ID: <20070903122747.54CBA8132@code0.codespeak.net> Author: arigo Date: Mon Sep 3 14:27:46 2007 New Revision: 46274 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py Log: More progress, mostly test fixing. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Mon Sep 3 14:27:46 2007 @@ -96,7 +96,7 @@ def spawnv_llimpl(mode, path, args): mode = rffi.cast(rffi.INT, mode) l_args = rffi.liststr2charpp(args) - childpid = os_spawnv(mode, l_path, l_args) + childpid = os_spawnv(mode, path, l_args) rffi.free_charpp(l_args) if childpid == -1: raise OSError(rffi.get_errno(), "os_spawnv failed") Modified: pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py Mon Sep 3 14:27:46 2007 @@ -599,7 +599,7 @@ return mask2 f1 = compile(does_stuff, []) res = f1() - assert res == 0660 + assert res == does_stuff() if hasattr(os, 'getpid'): def test_os_getpid(): @@ -834,12 +834,12 @@ compared_with.sort() assert result == compared_with -if hasattr(posix, 'execv'): +if hasattr(posix, 'execv') and hasattr(posix, 'fork'): def test_execv(): filename = str(udir.join('test_execv.txt')) def does_stuff(): progname = str(sys.executable) - l = [progname, '-c', 'open("%s","w").write("1")' % filename] + l = [progname, '-c', 'open(%r,"w").write("1")' % filename] pid = os.fork() if pid == 0: os.execv(progname, l) @@ -866,7 +866,7 @@ l = [] l.append(progname) l.append("-c") - l.append('import os; open("%s", "w").write(os.environ["STH"])' % filename) + l.append('import os; open(%r, "w").write(os.environ["STH"])' % filename) env = {} env["STH"] = "42" env["sthelse"] = "a" @@ -884,8 +884,17 @@ def test_spawnv(): filename = str(udir.join('test_spawnv.txt')) progname = str(sys.executable) + scriptpath = udir.join('test_spawnv.py') + scriptpath.write('f=open(%r,"w")\nf.write("2")\nf.close\n' % filename) + scriptname = str(scriptpath) def does_stuff(): - l = [progname, '-c', 'open("%s","w").write("2")' % filename] + # argument quoting on Windows is completely ill-defined. + # don't let yourself be fooled by the idea that if os.spawnv() + # takes a list of strings, then the receiving program will + # nicely see these strings as arguments with no further quote + # processing. Achieving this is nearly impossible - even + # CPython doesn't try at all. + l = [progname, scriptname] pid = os.spawnv(os.P_NOWAIT, progname, l) os.waitpid(pid, 0) func = compile(does_stuff, []) From antocuni at codespeak.net Mon Sep 3 16:03:24 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 16:03:24 +0200 (CEST) Subject: [pypy-svn] r46277 - in pypy/dist/pypy/translator: jvm/test oosupport Message-ID: <20070903140324.097DC8144@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 16:03:24 2007 New Revision: 46277 Modified: pypy/dist/pypy/translator/jvm/test/test_streamio.py pypy/dist/pypy/translator/jvm/test/test_tuple.py pypy/dist/pypy/translator/oosupport/constant.py Log: this extra DUP was needed for gencli, where dict of void are represented with generic classes, but not in genjvm, where they are plain classes. This code is not even longer needed in gencli, becase of the _check_for_void_dict in CLIDictMixin! Modified: pypy/dist/pypy/translator/jvm/test/test_streamio.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_streamio.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_streamio.py Mon Sep 3 16:03:24 2007 @@ -24,25 +24,5 @@ pass class TestTextOutputFilter(JvmTest, BaseTestTextOutputFilter): - def test_write_nl(self): - py.test.skip("VerifyError - Incompatible object arguments") - - def test_write_cr(self): - py.test.skip("VerifyError - Incompatible object arguments") - - def test_write_crnl(self): - py.test.skip("VerifyError - Incompatible object arguments") - - def test_write_tell_nl(self): - py.test.skip("VerifyError - Incompatible object arguments") - - def test_write_tell_cr(self): - py.test.skip("VerifyError - Incompatible object arguments") - - def test_write_tell_crnl(self): - py.test.skip("VerifyError - Incompatible object arguments") - - def test_write_seek(self): - py.test.skip("VerifyError - Incompatible object arguments") - + pass Modified: pypy/dist/pypy/translator/jvm/test/test_tuple.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_tuple.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_tuple.py Mon Sep 3 16:03:24 2007 @@ -10,14 +10,3 @@ assert res.item0 == 1.0 and res.item1 == 1 res = self.interpret(fn, [1.0, 1.0]) assert res.item0 == 1.0 and res.item1 == 1.0 - - #v116 = oosend(('ll_contains'), (), i_0) - # Arg 01: Exception in thread "main" java.lang.VerifyError - def test_constant_tuple_contains(self): - py.test.skip("VerifyError - Incompatible object argumemt") - - def test_constant_tuple_contains2(self): - py.test.skip("VerifyError - Incompatible object argumemt") - - def test_constant_unichar_tuple_contains(self): - py.test.skip("VerifyError - Incompatible object argumemt") Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Mon Sep 3 16:03:24 2007 @@ -648,11 +648,7 @@ gen.add_comment(' key=%r value=%r' % (key,value)) push_constant(self.db, KEYTYPE, key, gen) gen.prepare_generic_argument(KEYTYPE) - if VALUETYPE is ootype.Void: - # special case dict of Void; for now store the key as value? - gen.dup(KEYTYPE) - else: - push_constant(self.db, VALUETYPE, value, gen) + push_constant(self.db, VALUETYPE, value, gen) gen.prepare_generic_argument(VALUETYPE) gen.call_method(SELFTYPE, 'll_set') From antocuni at codespeak.net Mon Sep 3 16:08:37 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 16:08:37 +0200 (CEST) Subject: [pypy-svn] r46278 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20070903140837.D01338144@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 16:08:37 2007 New Revision: 46278 Modified: pypy/dist/pypy/translator/jvm/constant.py pypy/dist/pypy/translator/jvm/test/test_constant.py Log: don't try to access custom eq&hash functions on a null dict. Modified: pypy/dist/pypy/translator/jvm/constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/constant.py (original) +++ pypy/dist/pypy/translator/jvm/constant.py Mon Sep 3 16:08:37 2007 @@ -88,6 +88,8 @@ # key_eq is a Python function and graph is, well, a method # graph that seems to be added to the function pointer # somewhere. Adapted from cli/constant.py + if self.value is ootype.null(self.value._TYPE): + return self.eq_jcls = self.db.record_delegate_standalone_func_impl( self.value._dict.key_eq.graph) self.hash_jcls = self.db.record_delegate_standalone_func_impl( Modified: pypy/dist/pypy/translator/jvm/test/test_constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_constant.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_constant.py Mon Sep 3 16:08:37 2007 @@ -101,26 +101,25 @@ assert self.interpret(fn2, [1]) == True def test_customdict_circular(self): - py.test.skip("Circular dicts are not supported in JVM") -# from pypy.rlib.objectmodel import r_dict -# def key_eq(a, b): -# return a.x[0] == b.x[0] -# def key_hash(a): -# return ord(a.x[0]) -# -# class A: -# def __init__(self, x): -# self.x = x -# a = A('foo') -# a.dict = r_dict(key_eq, key_hash) -# a.dict[a] = 42 -# def fn(b): -# if b: -# s = A('foo') -# else: -# s = A('bar') -# return a.dict[s] -# assert self.interpret(fn, [True]) == 42 + from pypy.rlib.objectmodel import r_dict + def key_eq(a, b): + return a.x[0] == b.x[0] + def key_hash(a): + return ord(a.x[0]) + + class A: + def __init__(self, x): + self.x = x + a = A('foo') + a.dict = r_dict(key_eq, key_hash) + a.dict[a] = 42 + def fn(b): + if b: + s = A('foo') + else: + s = A('bar') + return a.dict[s] + assert self.interpret(fn, [True]) == 42 def test_multiple_step(self): from pypy.translator.oosupport import constant From antocuni at codespeak.net Mon Sep 3 16:13:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 16:13:00 +0200 (CEST) Subject: [pypy-svn] r46279 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070903141300.80D9D814D@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 16:13:00 2007 New Revision: 46279 Modified: pypy/dist/pypy/translator/jvm/test/test_dict.py Log: this test simply pass now Modified: pypy/dist/pypy/translator/jvm/test/test_dict.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_dict.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_dict.py Mon Sep 3 16:13:00 2007 @@ -6,11 +6,8 @@ def test_invalid_iterator(self): py.test.skip("test_invalid_iterator() doesn't work yet") - def test_dict_of_dict(self): - py.test.skip("hard to serialize a recursive dictionary") - def test_recursive(self): - py.test.skip("hard to serialize a recursive dictionary") + py.test.skip("JVM doesn't support recursive dicts") class TestJvmEmptyDict(JvmTest, oodict.BaseTestEmptyDict): def test_iterate_over_empty_dict(self): From arigo at codespeak.net Mon Sep 3 17:02:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 17:02:37 +0200 (CEST) Subject: [pypy-svn] r46280 - in pypy/branch/pypy-more-rtti-inprogress: rpython/module translator/c/test Message-ID: <20070903150237.1DBAE8168@code0.codespeak.net> Author: arigo Date: Mon Sep 3 17:02:35 2007 New Revision: 46280 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py Log: os.utime(): * Windows support * sub-second resolution support (not for Windows) Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Mon Sep 3 17:02:35 2007 @@ -19,6 +19,12 @@ from pypy.rpython.tool import rffi_platform as platform posix = __import__(os.name) +if sys.platform.startswith('win'): + underscore_on_windows = '_' +else: + underscore_on_windows = '' + + class CConfig: """ Definitions for platform integration. @@ -32,6 +38,7 @@ """ _includes_ = [] if not sys.platform.startswith('win'): + # XXX many of these includes are not portable at all _includes_ += ['dirent.h', 'sys/stat.h', 'sys/times.h', 'utime.h', 'sys/types.h', 'unistd.h', 'signal.h', 'sys/wait.h'] @@ -43,15 +50,17 @@ ('tms_stime', rffi.INT), ('tms_cutime', rffi.INT), ('tms_cstime', rffi.INT)]) + else: + _includes_ += ['sys/utime.h'] SEEK_SET = platform.DefinedConstantInteger('SEEK_SET') SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR') SEEK_END = platform.DefinedConstantInteger('SEEK_END') -if sys.platform.startswith('win'): - underscore_on_windows = '_' -else: - underscore_on_windows = '' + HAVE_UTIMES = platform.Has('utimes') + UTIMBUF = platform.Struct('struct '+underscore_on_windows+'utimbuf', + [('actime', rffi.INT), + ('modtime', rffi.INT)]) class RegisterOs(BaseLazyRegistering): @@ -133,29 +142,49 @@ @registering(os.utime) def register_os_utime(self): - TIME_T = rffi.INT # XXX do the right thing - UTIMEBUFP = rffi.CStructPtr('utimbuf', ('actime', TIME_T), - ('modtime', TIME_T)) + UTIMBUFP = lltype.Ptr(self.UTIMBUF) + os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) - # XXX sys/types.h is not portable at all - os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMEBUFP], rffi.INT) + if self.HAVE_UTIMES: + class CConfig: + TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.LONG), + ('tv_usec', rffi.LONG)]) + config = platform.configure(CConfig) + TIMEVAL = config['TIMEVAL'] + TIMEVAL2P = lltype.Ptr(lltype.FixedSizeArray(TIMEVAL, 2)) + os_utimes = self.llexternal('utimes', [rffi.CCHARP, TIMEVAL2P], rffi.INT) + + def os_utime_platform(path, actime, modtime): + import math + l_times = lltype.malloc(TIMEVAL2P.TO, flavor='raw') + fracpart, intpart = math.modf(actime) + l_times[0].c_tv_sec = int(intpart) + l_times[0].c_tv_usec = int(fracpart * 1E6) + fracpart, intpart = math.modf(modtime) + l_times[1].c_tv_sec = int(intpart) + l_times[1].c_tv_usec = int(fracpart * 1E6) + error = os_utimes(path, l_times) + lltype.free(l_times, flavor='raw') + return error + else: + # we only have utime(), which does not allow sub-second resolution + def os_utime_platform(path, actime, modtime): + l_utimbuf = lltype.malloc(UTIMBUFP.TO, flavor='raw') + l_utimbuf.c_actime = int(actime) + l_utimbuf.c_modtime = int(modtime) + error = os_utime(path, l_utimbuf) + lltype.free(l_utimbuf, flavor='raw') + return error def os_utime_llimpl(path, tp): - # XXX right now they're all ints, might change in future - # XXX does not use utimes, even when available # NB. this function is specialized; we get one version where # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if tp is None: - l_utimebuf = lltype.nullptr(UTIMEBUFP.TO) + error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) else: - l_utimebuf = lltype.malloc(UTIMEBUFP.TO, flavor='raw') actime, modtime = tp - l_utimebuf.c_actime = int(actime) - l_utimebuf.c_modtime = int(modtime) - error = os_utime(path, l_utimebuf) - if tp is not None: - lltype.free(l_utimebuf, flavor='raw') + error = os_utime_platform(path, actime, modtime) if error == -1: raise OSError(rffi.get_errno(), "os_utime failed") os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' Modified: pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py Mon Sep 3 17:02:35 2007 @@ -902,7 +902,6 @@ assert open(filename).read() == "2" def test_utime(): - # XXX utimes & float support path = str(udir.ensure("test_utime.txt")) from time import time, sleep t0 = time() From arigo at codespeak.net Mon Sep 3 17:07:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 17:07:15 +0200 (CEST) Subject: [pypy-svn] r46281 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903150715.BE48A8168@code0.codespeak.net> Author: arigo Date: Mon Sep 3 17:07:15 2007 New Revision: 46281 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Log: Fix os.utime() for Linux. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Mon Sep 3 17:07:15 2007 @@ -57,7 +57,6 @@ SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR') SEEK_END = platform.DefinedConstantInteger('SEEK_END') - HAVE_UTIMES = platform.Has('utimes') UTIMBUF = platform.Struct('struct '+underscore_on_windows+'utimbuf', [('actime', rffi.INT), ('modtime', rffi.INT)]) @@ -145,14 +144,22 @@ UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) - if self.HAVE_UTIMES: + class CConfig: + _includes_ = ['sys/time.h'] + HAVE_UTIMES = platform.Has('utimes') + config = platform.configure(CConfig) + + if config['HAVE_UTIMES']: class CConfig: + _includes_ = ['sys/time.h'] TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.LONG), ('tv_usec', rffi.LONG)]) config = platform.configure(CConfig) TIMEVAL = config['TIMEVAL'] TIMEVAL2P = lltype.Ptr(lltype.FixedSizeArray(TIMEVAL, 2)) - os_utimes = self.llexternal('utimes', [rffi.CCHARP, TIMEVAL2P], rffi.INT) + os_utimes = self.llexternal('utimes', [rffi.CCHARP, TIMEVAL2P], + rffi.INT, + includes=['sys/time.h']) def os_utime_platform(path, actime, modtime): import math From antocuni at codespeak.net Mon Sep 3 17:14:12 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 17:14:12 +0200 (CEST) Subject: [pypy-svn] r46282 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20070903151412.78A9B8168@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 17:14:11 2007 New Revision: 46282 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/test/test_dict.py Log: bugfix && test passes Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Mon Sep 3 17:14:11 2007 @@ -712,6 +712,8 @@ """ Loads from jvm slot #varidx, which is expected to hold a value of type vartype """ assert varidx < self.curfunc.next_offset + if jvartype is jVoid: + return opc = LOAD.for_type(jvartype) self.add_comment(" load_jvm_jar: jvartype=%s varidx=%s" % ( repr(jvartype), repr(varidx))) Modified: pypy/dist/pypy/translator/jvm/test/test_dict.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_dict.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_dict.py Mon Sep 3 17:14:11 2007 @@ -10,8 +10,7 @@ py.test.skip("JVM doesn't support recursive dicts") class TestJvmEmptyDict(JvmTest, oodict.BaseTestEmptyDict): - def test_iterate_over_empty_dict(self): - py.test.skip("Iteration over empty dict is not supported, yet") + pass class TestJvmConstantDict(JvmTest, oodict.BaseTestConstantDict): pass From antocuni at codespeak.net Mon Sep 3 17:21:48 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 17:21:48 +0200 (CEST) Subject: [pypy-svn] r46283 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070903152148.E1E32816F@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 17:21:48 2007 New Revision: 46283 Modified: pypy/dist/pypy/translator/jvm/test/test_string.py Log: horray, these tests pass for free! :-) Modified: pypy/dist/pypy/translator/jvm/test/test_string.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_string.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_string.py Mon Sep 3 17:21:48 2007 @@ -14,16 +14,10 @@ test_char_unichar_eq = test_unichar_const test_char_unichar_eq_2 = test_unichar_const - def test_convert_char_to_unichar(self): - py.test.skip("Fails with arguments, Expected to find an object/array on the stack") - def test_upper(self): py.test.skip("eval has trouble with evaluation of null literals") test_lower = test_upper - def test_float(self): - py.test.skip("JVM does not yet support ooparse_float") - def test_getitem_exc(self): # This test is supposed to crash in a system specific way; # in our case an StringIndexOutOfBounds exception is thrown, From arigo at codespeak.net Mon Sep 3 17:21:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 17:21:57 +0200 (CEST) Subject: [pypy-svn] r46284 - pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem Message-ID: <20070903152157.ED2B08177@code0.codespeak.net> Author: arigo Date: Mon Sep 3 17:21:57 2007 New Revision: 46284 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py Log: Temporary workaround for an annotation bug. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py Mon Sep 3 17:21:57 2007 @@ -54,6 +54,10 @@ unrolling_arg_tps = unrolling_iterable(enumerate(args)) def wrapper(*args): + # XXX the next line is a workaround for the annotation bug + # shown in rpython.test.test_llann:test_pbctype. Remove it + # when the test is fixed... + assert isinstance(lltype.Signed, lltype.Number) real_args = () if stringpolicy == 'fullauto': to_free = () From antocuni at codespeak.net Mon Sep 3 17:38:59 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 17:38:59 +0200 (CEST) Subject: [pypy-svn] r46285 - in pypy/dist/pypy/translator: cli/test jvm/test oosupport/test_template Message-ID: <20070903153859.D70C68172@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 17:38:59 2007 New Revision: 46285 Added: pypy/dist/pypy/translator/oosupport/test_template/class_.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/test/test_class.py pypy/dist/pypy/translator/jvm/test/test_class.py Log: move some tests shared by cli and jvm to oosupport. More jvm tests pass. Modified: pypy/dist/pypy/translator/cli/test/test_class.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_class.py (original) +++ pypy/dist/pypy/translator/cli/test/test_class.py Mon Sep 3 17:38:59 2007 @@ -1,69 +1,9 @@ import py from pypy.translator.cli.test.runtest import CliTest -from pypy.rpython.test.test_rclass import BaseTestRclass -from pypy.rpython.test.test_rspecialcase import BaseTestRspecialcase +from pypy.translator.oosupport.test_template.class_ import BaseTestClass, BaseTestSpecialcase -class TestCliClass(CliTest, BaseTestRclass): - def test_abstract_method(self): - class Base: - pass - class A(Base): - def f(self, x): - return x+1 - class B(Base): - def f(self, x): - return x+2 - def call(obj, x): - return obj.f(x) - def fn(x): - a = A() - b = B() - return call(a, x) + call(b, x) - assert self.interpret(fn, [0]) == 3 - - def test_abstract_method2(self): - class Root: - pass - class Class1(Root): - pass - class Derived(Class1): - x = 1 - class Class2(Root): - x = 2 - def get_x(obj): - return obj.x - def fn(): - a = Derived() - b = Class2() - return get_x(a) + get_x(b) - assert self.interpret(fn, []) == 3 - - def test_same_name(self): - class A: - def __init__(self, x): - self.x = x - B=A - class A: - def __init__(self, y): - self.y = y - assert A is not B - assert A.__name__ == B.__name__ - def fn(x, y): - obj1 = B(x) - obj2 = A(y) - return obj1.x + obj2.y - assert self.interpret(fn, [1, 2]) == 3 - - def test_ctr_location(self): - class A: - _annspecialcase_ = "specialize:ctr_location" - def __init__(self, x): - self.x = x - def fn(x, y): - a = A(x) - b = A(y) - return a.x + b.x - assert self.interpret(fn, [1, 2]) == 3 +class TestCliClass(CliTest, BaseTestClass): + pass -class TestCliSpecialCase(CliTest, BaseTestRspecialcase): +class TestCliSpecialCase(CliTest, BaseTestSpecialcase): pass Modified: pypy/dist/pypy/translator/jvm/test/test_class.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_class.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_class.py Mon Sep 3 17:38:59 2007 @@ -1,52 +1,10 @@ import py from pypy.translator.jvm.test.runtest import JvmTest -from pypy.rpython.test.test_rclass import BaseTestRclass -from pypy.rpython.test.test_rspecialcase import BaseTestRspecialcase +from pypy.translator.oosupport.test_template.class_ import BaseTestClass, BaseTestSpecialcase -class TestJvmClass(JvmTest, BaseTestRclass): +class TestJvmClass(JvmTest, BaseTestClass): def test_overridden_classattr_as_defaults(self): py.test.skip("JVM doesn't support overridden default value yet") - def test_abstract_method(self): - class Base(object): - pass - class A(Base): - def f(self, x): - return x+1 - class B(Base): - def f(self, x): - return x+2 - def call(obj, x): - return obj.f(x) - def fn(x): - a = A() - b = B() - return call(a, x) + call(b, x) - assert self.interpret(fn, [0]) == 3 - - def test_abstract_method2(self): - class Root(object): - pass - class Class1(Root): - pass - class Derived(Class1): - x = 1 - class Class2(Root): - x = 2 - def get_x(obj): - return obj.x - def fn(): - derived = Derived() - cls2 = Class2() - return get_x(derived) + get_x(cls2) - assert self.interpret(fn, []) == 3 - - def test_same_name(self): - py.test.skip("JVM doesn't support classes with the same name") - - def test_ctr_location(self): - py.test.skip("Ask cuni if this applies to JVM -I don't think so") - - -#class TestCliSpecialCase(CliTest, BaseTestRspecialcase): -# pass +class TestJvmSpecialCase(JvmTest, BaseTestSpecialcase): + pass Added: pypy/dist/pypy/translator/oosupport/test_template/class_.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/test_template/class_.py Mon Sep 3 17:38:59 2007 @@ -0,0 +1,68 @@ +import py +from pypy.rpython.test.test_rclass import BaseTestRclass +from pypy.rpython.test.test_rspecialcase import BaseTestRspecialcase + +class BaseTestClass(BaseTestRclass): + def test_abstract_method(self): + class Base: + pass + class A(Base): + def f(self, x): + return x+1 + class B(Base): + def f(self, x): + return x+2 + def call(obj, x): + return obj.f(x) + def fn(x): + a = A() + b = B() + return call(a, x) + call(b, x) + assert self.interpret(fn, [0]) == 3 + + def test_abstract_method2(self): + class Root: + pass + class Class1(Root): + pass + class Derived(Class1): + x = 1 + class Class2(Root): + x = 2 + def get_x(obj): + return obj.x + def fn(): + a = Derived() + b = Class2() + return get_x(a) + get_x(b) + assert self.interpret(fn, []) == 3 + + def test_same_name(self): + class A: + def __init__(self, x): + self.x = x + B=A + class A: + def __init__(self, y): + self.y = y + assert A is not B + assert A.__name__ == B.__name__ + def fn(x, y): + obj1 = B(x) + obj2 = A(y) + return obj1.x + obj2.y + assert self.interpret(fn, [1, 2]) == 3 + + def test_ctr_location(self): + class A: + _annspecialcase_ = "specialize:ctr_location" + def __init__(self, x): + self.x = x + def fn(x, y): + a = A(x) + b = A(y) + return a.x + b.x + assert self.interpret(fn, [1, 2]) == 3 + +class BaseTestSpecialcase(BaseTestRspecialcase): + pass From arigo at codespeak.net Mon Sep 3 17:55:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 17:55:31 +0200 (CEST) Subject: [pypy-svn] r46286 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903155531.B0BC8816F@code0.codespeak.net> Author: arigo Date: Mon Sep 3 17:55:29 2007 New Revision: 46286 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Log: Fix os.listdir() on Linux Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Mon Sep 3 17:55:29 2007 @@ -620,16 +620,20 @@ else: class CConfig: - DIRENT = platform.Struct('dirent', + _includes_ = ['sys/types.h', 'dirent.h'] + DIRENT = platform.Struct('struct dirent', [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]) config = platform.configure(CConfig) DIRENT = config['DIRENT'] DIRENTP = lltype.Ptr(DIRENT) DIRP = rffi.COpaquePtr('DIR') - os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP) - os_readdir = self.llexternal('readdir', [DIRP], DIRENTP) - os_closedir = self.llexternal('closedir', [DIRP], rffi.INT) + os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP, + includes=CConfig._includes_) + os_readdir = self.llexternal('readdir', [DIRP], DIRENTP, + includes=CConfig._includes_) + os_closedir = self.llexternal('closedir', [DIRP], rffi.INT, + includes=CConfig._includes_) def os_listdir_llimpl(path): dirp = os_opendir(path) From mwh at codespeak.net Mon Sep 3 18:05:24 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 3 Sep 2007 18:05:24 +0200 (CEST) Subject: [pypy-svn] r46287 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903160524.6BA248177@code0.codespeak.net> Author: mwh Date: Mon Sep 3 18:05:23 2007 New Revision: 46287 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py Log: os.name is not a good way of distinguishing os x... Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py Mon Sep 3 18:05:23 2007 @@ -1,4 +1,4 @@ -import os +import os, sys from pypy.annotation import model as annmodel from pypy.rpython.controllerentry import Controller from pypy.rpython.extfunc import register_external @@ -127,7 +127,7 @@ # ____________________________________________________________ # Access to the 'environ' external variable -if os.name.startswith('darwin'): +if sys.platform.startswith('darwin'): CCHARPPP = lltype.Ptr(lltype.FixedSizeArray(rffi.CCHARPP, 1)) _os_NSGetEnviron = rffi.llexternal('_NSGetEnviron', [], CCHARPPP, includes=['crt_externs.h']) From arigo at codespeak.net Mon Sep 3 18:26:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 18:26:46 +0200 (CEST) Subject: [pypy-svn] r46288 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903162646.F24F08174@code0.codespeak.net> Author: arigo Date: Mon Sep 3 18:26:46 2007 New Revision: 46288 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py Log: Attempted fix for OS/X Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_environ.py Mon Sep 3 18:26:46 2007 @@ -104,11 +104,15 @@ if hasattr(__import__(os.name), 'unsetenv'): - os_unsetenv = rffi.llexternal('unsetenv', [rffi.CCHARP], rffi.INT) + if sys.platform.startswith('darwin'): + RETTYPE = lltype.Void + else: + RETTYPE = rffi.INT + os_unsetenv = rffi.llexternal('unsetenv', [rffi.CCHARP], RETTYPE) def unsetenv_llimpl(name): l_name = rffi.str2charp(name) - error = os_unsetenv(l_name) + error = os_unsetenv(l_name) # 'error' is None on OS/X rffi.free_charp(l_name) if error: raise OSError(rffi.get_errno(), "os_unsetenv failed") From arigo at codespeak.net Mon Sep 3 18:26:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 18:26:56 +0200 (CEST) Subject: [pypy-svn] r46289 - pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem Message-ID: <20070903162656.408318180@code0.codespeak.net> Author: arigo Date: Mon Sep 3 18:26:55 2007 New Revision: 46289 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py Log: Attempted fix for Windows Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py Mon Sep 3 18:26:55 2007 @@ -418,7 +418,13 @@ # __________ the standard C library __________ if sys.platform == 'win32': - standard_c_lib = ctypes.cdll.LoadLibrary('msvcrt.dll') + # trying to guess the correct libc... only a few tests fail if there + # is a mismatch between the one used by python2x.dll and the one + # loaded here + if sys.version_info < (2, 4): + standard_c_lib = ctypes.cdll.LoadLibrary('msvcrt.dll') + else: + standard_c_lib = ctypes.cdll.LoadLibrary('msvcr71.dll') else: standard_c_lib = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c')) From arigo at codespeak.net Mon Sep 3 18:28:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 18:28:37 +0200 (CEST) Subject: [pypy-svn] r46290 - pypy/branch/pypy-more-rtti-inprogress/rpython/module Message-ID: <20070903162837.6AC8C8180@code0.codespeak.net> Author: arigo Date: Mon Sep 3 18:28:37 2007 New Revision: 46290 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Log: Sprinkling underscores in front of C library function names on Windows, for ctypes to find the functions. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Mon Sep 3 18:28:37 2007 @@ -115,7 +115,7 @@ @registering(os.dup) def register_os_dup(self): - os_dup = self.llexternal('dup', [rffi.INT], rffi.INT) + os_dup = self.llexternal(underscore_on_windows+'dup', [rffi.INT], rffi.INT) def dup_llimpl(fd): newfd = rffi.cast(lltype.Signed, os_dup(rffi.cast(rffi.INT, fd))) @@ -128,7 +128,8 @@ @registering(os.dup2) def register_os_dup2(self): - os_dup2 = self.llexternal('dup2', [rffi.INT, rffi.INT], rffi.INT) + os_dup2 = self.llexternal(underscore_on_windows+'dup2', + [rffi.INT, rffi.INT], rffi.INT) def dup2_llimpl(fd, newfd): error = rffi.cast(lltype.Signed, os_dup2(rffi.cast(rffi.INT, fd), @@ -356,7 +357,8 @@ @registering(os.open) def register_os_open(self): - os_open = self.llexternal('open', [rffi.CCHARP, rffi.INT, rffi.MODE_T], + os_open = self.llexternal(underscore_on_windows+'open', + [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) def os_open_llimpl(path, flags, mode): @@ -375,7 +377,8 @@ @registering(os.read) def register_os_read(self): - os_read = self.llexternal('read', [rffi.INT, rffi.VOIDP, rffi.SIZE_T], + os_read = self.llexternal(underscore_on_windows+'read', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SIZE_T) def os_read_llimpl(fd, count): @@ -401,8 +404,9 @@ @registering(os.write) def register_os_write(self): - os_write = self.llexternal('write', [rffi.INT, rffi.VOIDP, - rffi.SIZE_T], rffi.SIZE_T) + os_write = self.llexternal(underscore_on_windows+'write', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T], + rffi.SIZE_T) def os_write_llimpl(fd, data): count = len(data) @@ -428,7 +432,7 @@ @registering(os.close) def register_os_close(self): - os_close = self.llexternal('close', [rffi.INT], rffi.INT) + os_close = self.llexternal(underscore_on_windows+'close', [rffi.INT], rffi.INT) def close_llimpl(fd): error = rffi.cast(lltype.Signed, os_close(rffi.cast(rffi.INT, fd))) @@ -781,7 +785,7 @@ @registering(os.isatty) def register_os_isatty(self): - os_isatty = self.llexternal('isatty', [rffi.INT], rffi.INT) + os_isatty = self.llexternal(underscore_on_windows+'isatty', [rffi.INT], rffi.INT) def isatty_llimpl(fd): res = os_isatty(rffi.cast(rffi.INT, fd)) @@ -816,7 +820,7 @@ @registering(os.unlink) def register_os_unlink(self): - os_unlink = self.llexternal('unlink', [rffi.CCHARP], rffi.INT) + os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = os_unlink(pathname) @@ -828,7 +832,7 @@ @registering(os.chdir) def register_os_chdir(self): - os_chdir = self.llexternal('chdir', [rffi.CCHARP], rffi.INT) + os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) def chdir_llimpl(path): res = os_chdir(path) @@ -844,7 +848,8 @@ ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal('mkdir', [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(underscore_on_windows+'mkdir', + [rffi.CCHARP]+ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -860,7 +865,7 @@ @registering(os.rmdir) def register_os_rmdir(self): - os_rmdir = self.llexternal('rmdir', [rffi.CCHARP], rffi.INT) + os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = os_rmdir(pathname) @@ -872,7 +877,7 @@ @registering(os.chmod) def register_os_chmod(self): - os_chmod = self.llexternal('chmod', [rffi.CCHARP, rffi.MODE_T], + os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): @@ -898,7 +903,7 @@ @registering(os.umask) def register_os_umask(self): - os_umask = self.llexternal('umask', [rffi.MODE_T], rffi.MODE_T) + os_umask = self.llexternal(underscore_on_windows+'umask', [rffi.MODE_T], rffi.MODE_T) def umask_llimpl(fd): res = os_umask(rffi.cast(rffi.MODE_T, fd)) From arigo at codespeak.net Mon Sep 3 18:32:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 Sep 2007 18:32:31 +0200 (CEST) Subject: [pypy-svn] r46291 - pypy/branch/pypy-more-rtti-inprogress/rpython/module/test Message-ID: <20070903163231.151618180@code0.codespeak.net> Author: arigo Date: Mon Sep 3 18:32:30 2007 New Revision: 46291 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_posix.py Log: Fix tests. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_posix.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_posix.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_posix.py Mon Sep 3 18:32:30 2007 @@ -104,20 +104,19 @@ res = self.interpret(f,[fi]) raises( OSError, os.fstat, fi) - def test_ftruncate(self): - def f(fi,len): - posix.ftruncate(fi,len) - fi = os.open(path,os.O_RDWR,0777) - func = self.interpret(f,[fi,6]) - assert os.fstat(fi).st_size == 6 - - def test_getuid(self): - def f(): - return os.getuid() - assert self.interpret(f, []) == f() - -if not hasattr(os, 'ftruncate'): - del BaseTestPosix.test_ftruncate + if hasattr(os, 'ftruncate'): + def test_ftruncate(self): + def f(fi,len): + os.ftruncate(fi,len) + fi = os.open(path,os.O_RDWR,0777) + func = self.interpret(f,[fi,6]) + assert os.fstat(fi).st_size == 6 + + if hasattr(os, 'getuid'): + def test_getuid(self): + def f(): + return os.getuid() + assert self.interpret(f, []) == f() class TestLLtype(BaseTestPosix, LLRtypeMixin): if False and hasattr(os, 'uname'): From antocuni at codespeak.net Mon Sep 3 22:24:38 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 22:24:38 +0200 (CEST) Subject: [pypy-svn] r46294 - in pypy/dist/pypy/translator: cli jvm oosupport Message-ID: <20070903202438.484B98146@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 22:24:36 2007 New Revision: 46294 Modified: pypy/dist/pypy/translator/cli/ilgenerator.py pypy/dist/pypy/translator/cli/metavm.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/oosupport/metavm.py Log: refactor metavm._Call: now most of the logic is shared between jvm and cli Modified: pypy/dist/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/dist/pypy/translator/cli/ilgenerator.py (original) +++ pypy/dist/pypy/translator/cli/ilgenerator.py Mon Sep 3 22:24:36 2007 @@ -308,6 +308,10 @@ def emit(self, instr, *args): self.ilasm.opcode(instr, *args) + def call_primitive(self, op, module, name): + func_name = '[pypylib]pypy.builtin.%s::%s' % (module, name) + self.call_op(op, func_name) + def call_graph(self, graph, func_name=None): if func_name is None: # else it is a suggested primitive self.db.pending_function(graph) Modified: pypy/dist/pypy/translator/cli/metavm.py ============================================================================== --- pypy/dist/pypy/translator/cli/metavm.py (original) +++ pypy/dist/pypy/translator/cli/metavm.py Mon Sep 3 22:24:36 2007 @@ -2,36 +2,21 @@ from pypy.rpython.ootypesystem import ootype from pypy.translator.oosupport.metavm import Generator, InstructionList, MicroInstruction,\ PushAllArgs, StoreResult, GetField, SetField, DownCast +from pypy.translator.oosupport.metavm import _Call as _OOCall from pypy.translator.cli.comparer import EqualityComparer from pypy.translator.cli.cts import WEAKREF from pypy.translator.cli.dotnet import _static_meth, NativeInstance STRING_HELPER_CLASS = '[pypylib]pypy.runtime.String' -class _Call(MicroInstruction): +class _Call(_OOCall): + def render(self, generator, op): callee = op.args[0].value if isinstance(callee, _static_meth): self._render_native_function(generator, callee, op.args) - elif hasattr(callee, "graph"): - graph = callee.graph - method_name = oopspec.get_method_name(graph, op) - if method_name is None: - self._render_function(generator, graph, op.args) - else: - self._render_method(generator, method_name, op.args[1:]) else: - self._render_primitive_function(generator, callee, op) - - - def _load_arg_or_null(self, generator, arg): - if arg.concretetype is ootype.Void: - if arg.value is None: - generator.ilasm.opcode('ldnull') # special-case: use None as a null value - else: - assert False, "Don't know how to load this arg" - else: - generator.load(arg) + _OOCall.render(self, generator, op) def _render_native_function(self, generator, funcdesc, args): for func_arg in args[1:]: # push parameters @@ -45,17 +30,20 @@ signature = '%s %s::%s(%s)' % (ret_type, funcdesc._cls._name, funcdesc._name, arg_list) generator.call_signature(signature) - def _render_function(self, generator, graph, args): - primitive = getattr(graph.func, 'suggested_primitive', False) - for func_arg in args[1:]: # push parameters - generator.load(func_arg) - - if primitive: - _, module = graph.func.__module__.rsplit('.', 1) - func_name = '[pypylib]pypy.builtin.%s::%s' % (module, graph.func.func_name) - generator.call_graph(graph, func_name) + def _load_arg_or_null(self, generator, arg): + if arg.concretetype is ootype.Void: + if arg.value is None: + generator.ilasm.opcode('ldnull') # special-case: use None as a null value + else: + assert False, "Don't know how to load this arg" else: - generator.call_graph(graph) + generator.load(arg) + + +class _CallMethod(_Call): + def render(self, generator, op): + method = op.args[0] + self._render_method(generator, method.value, op.args[1:]) def _render_method(self, generator, method_name, args): this = args[0] @@ -95,20 +83,8 @@ method_name == 'll_current_key')): generator.ilasm.pop() - def _render_primitive_function(self, generator, callee, op): - for func_arg in op.args[1:]: # push parameters - self._load_arg_or_null(generator, func_arg) - module, name = callee._name.split(".") - func_name = '[pypylib]pypy.builtin.%s::%s' % (module, name) - generator.call_op(op, func_name) - -class _CallMethod(_Call): - def render(self, generator, op): - method = op.args[0] - self._render_method(generator, method.value, op.args[1:]) - -class _IndirectCall(_Call): +class _IndirectCall(_CallMethod): def render(self, generator, op): # discard the last argument because it's used only for analysis self._render_method(generator, 'Invoke', op.args[:-1]) Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Mon Sep 3 22:24:36 2007 @@ -1014,7 +1014,8 @@ # we have to "deal with it" self.prepare_generic_result(RETTYPE) - def call_primitive(self, graph): + def call_primitive(self, op, module, name): + graph = op.args[0].value # XXX argtypes, rettype = self.db.types_for_graph(graph) mthd = Method.s(jPyPy, graph.func.func_name, argtypes, rettype) self.emit(mthd) Modified: pypy/dist/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/metavm.py (original) +++ pypy/dist/pypy/translator/oosupport/metavm.py Mon Sep 3 22:24:36 2007 @@ -172,7 +172,7 @@ Stack: argN, arg2, arg1, this, ... -> ret, ... """ raise NotImplementedError - def call_primitive(self, graph): + def call_primitive(self, op, module, name): """ Like call_graph, but it has been suggested that the method be rendered as a primitive. @@ -395,23 +395,34 @@ def render(self, generator, op): generator.branch_conditionally(False, self.label) + class _Call(MicroInstruction): + + def _get_primitive_name(self, callee): + try: + graph = callee.graph + except AttributeError: + return callee._name.rsplit('.', 1) + else: + if getattr(graph.func, 'suggested_primitive', False): + _, module = graph.func.__module__.rsplit('.', 1) + return module, graph.func.func_name + else: + return None + def render(self, generator, op): callee = op.args[0].value - graph = callee.graph - method_name = None # XXX oopspec.get_method_name(graph, op) + is_primitive = self._get_primitive_name(callee) for arg in op.args[1:]: generator.load(arg) - if method_name is None: - if getattr(graph.func, 'suggested_primitive', False): - generator.call_primitive(graph) - else: - generator.call_graph(graph) + if is_primitive: + module, name = is_primitive + generator.call_primitive(op, module, name) else: - this = op.args[1] - generator.call_method(this.concretetype, method_name) + generator.call_graph(callee.graph) + class _CallMethod(MicroInstruction): def render(self, generator, op): From antocuni at codespeak.net Mon Sep 3 22:37:28 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 Sep 2007 22:37:28 +0200 (CEST) Subject: [pypy-svn] r46295 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20070903203728.A51F58149@code0.codespeak.net> Author: antocuni Date: Mon Sep 3 22:37:28 2007 New Revision: 46295 Modified: pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/test/test_builtin.py Log: correctly implement generator.call_primitive. Implement ll_math.{fmod,floor}. Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Mon Sep 3 22:37:28 2007 @@ -119,12 +119,15 @@ then the return would be: ( (jString, jString), jBool ) """ - argtypes = [arg.concretetype for arg in graph.getargs() - if arg.concretetype is not ootype.Void] - jargtypes = tuple([self.lltype_to_cts(argty) for argty in argtypes]) - rettype = graph.getreturnvar().concretetype - jrettype = self.lltype_to_cts(rettype) - return jargtypes, jrettype + ARGS = [v.concretetype for v in graph.getargs()] + RESULT = graph.getreturnvar().concretetype + return self.types_for_signature(ARGS, RESULT) + + def types_for_signature(self, ARGS, RESULT): + ARGS = [ARG for ARG in ARGS if ARG is not ootype.Void] + jargtypes = tuple([self.lltype_to_cts(ARG) for ARG in ARGS]) + jrettype = self.lltype_to_cts(RESULT) + return jargtypes, jrettype def _function_for_graph(self, classobj, funcnm, is_static, graph): Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Mon Sep 3 22:37:28 2007 @@ -1015,9 +1015,9 @@ self.prepare_generic_result(RETTYPE) def call_primitive(self, op, module, name): - graph = op.args[0].value # XXX - argtypes, rettype = self.db.types_for_graph(graph) - mthd = Method.s(jPyPy, graph.func.func_name, argtypes, rettype) + callee = op.args[0].value + argtypes, rettype = self.db.types_for_signature(callee._TYPE.ARGS, callee._TYPE.RESULT) + mthd = Method.s(jPyPy, name, argtypes, rettype) self.emit(mthd) def call_oostring(self, OOTYPE): Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Mon Sep 3 22:37:28 2007 @@ -884,6 +884,19 @@ } // ---------------------------------------------------------------------- + // ll_math + + public static double ll_math_floor(double x) + { + return Math.floor(x); + } + + public static double ll_math_fmod(double x, double y) + { + return x % y; + } + + // ---------------------------------------------------------------------- // Convenient Helpers for throwing exceptions // // Also, an abstraction barrier: at a later date we may want to Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Mon Sep 3 22:37:28 2007 @@ -27,13 +27,7 @@ def test_os_isdir(self): py.test.skip("ll_os_stat is not currently implemented in the Jvm backed") - - def test_builtin_math_floor(self): - py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") - - def test_builtin_math_fmod(self): - py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") - + def test_builtin_math_frexp(self): py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") From pypy-svn at codespeak.net Tue Sep 4 06:33:27 2007 From: pypy-svn at codespeak.net (Loretta Mcgowan) Date: Tue, 35 Aug 2007 05:33:27 +0100 Subject: [pypy-svn] Pharmacy Shop Message-ID: <483541716.04632409696056@codespeak.net> New pharmacy shop: http://selfoccur.com From arigo at codespeak.net Tue Sep 4 12:47:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 4 Sep 2007 12:47:39 +0200 (CEST) Subject: [pypy-svn] r46300 - pypy/branch/pypy-more-rtti-inprogress/module/posix/test Message-ID: <20070904104739.24C34817D@code0.codespeak.net> Author: arigo Date: Tue Sep 4 12:47:37 2007 New Revision: 46300 Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py Log: Skip tests for Windows. Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py Tue Sep 4 12:47:37 2007 @@ -26,8 +26,9 @@ cls.w_path = space.wrap(str(path)) cls.w_path2 = space.wrap(str(path2)) cls.w_pdir = space.wrap(str(pdir)) - cls.w_getuid = space.wrap(os.getuid()) - cls.w_geteuid = space.wrap(os.geteuid()) + if hasattr(os, 'getuid'): + cls.w_getuid = space.wrap(os.getuid()) + cls.w_geteuid = space.wrap(os.geteuid()) def test_posix_is_pypy_s(self): assert self.posix.__file__ @@ -268,10 +269,11 @@ assert isinstance(i, str) assert isinstance(res, tuple) - def test_os_getuid(self): - os = self.posix - assert os.getuid() == self.getuid - assert os.geteuid() == self.geteuid + if hasattr(os, 'getuid'): + def test_os_getuid(self): + os = self.posix + assert os.getuid() == self.getuid + assert os.geteuid() == self.geteuid def test_largefile(self): os = self.posix From arigo at codespeak.net Tue Sep 4 13:02:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 4 Sep 2007 13:02:41 +0200 (CEST) Subject: [pypy-svn] r46301 - pypy/dist/lib-python/modified-2.4.1 Message-ID: <20070904110241.A85BD8177@code0.codespeak.net> Author: arigo Date: Tue Sep 4 13:02:40 2007 New Revision: 46301 Modified: pypy/dist/lib-python/modified-2.4.1/pickle.py Log: Avoid importing 're' from 'pickle'. This makes some tests running on top of py.py incredibly faster. Modified: pypy/dist/lib-python/modified-2.4.1/pickle.py ============================================================================== --- pypy/dist/lib-python/modified-2.4.1/pickle.py (original) +++ pypy/dist/lib-python/modified-2.4.1/pickle.py Tue Sep 4 13:02:40 2007 @@ -32,7 +32,6 @@ import marshal import sys import struct -import re import warnings __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", @@ -163,7 +162,7 @@ _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3] -__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)]) +__all__.extend([x for x in dir() if x[0].isalpha() and x == x.upper()]) del x From arigo at codespeak.net Tue Sep 4 13:04:55 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 4 Sep 2007 13:04:55 +0200 (CEST) Subject: [pypy-svn] r46302 - pypy/branch/pypy-more-rtti-inprogress/module/posix/test Message-ID: <20070904110455.BAB33817B@code0.codespeak.net> Author: arigo Date: Tue Sep 4 13:04:55 2007 New Revision: 46302 Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py Log: Skip on Windows. Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py Tue Sep 4 13:04:55 2007 @@ -255,10 +255,11 @@ exec code.compile() in d locals()['test_' + name] = d['test_wstar'] - def test_wifsignaled(self): - os = self.posix - assert os.WIFSIGNALED(0) == False - assert os.WIFSIGNALED(1) == True + if hasattr(os, 'WIFSIGNALED'): + def test_wifsignaled(self): + os = self.posix + assert os.WIFSIGNALED(0) == False + assert os.WIFSIGNALED(1) == True def test_os_uname(self): skip("Uname broken") From arigo at codespeak.net Tue Sep 4 13:39:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 4 Sep 2007 13:39:51 +0200 (CEST) Subject: [pypy-svn] r46303 - in pypy/branch/pypy-more-rtti-inprogress/rpython/tool: . test Message-ID: <20070904113951.4775E8177@code0.codespeak.net> Author: arigo Date: Tue Sep 4 13:39:50 2007 New Revision: 46303 Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/tool/rffi_platform.py pypy/branch/pypy-more-rtti-inprogress/rpython/tool/test/test_rffi_platform.py Log: If a CConfig has only 'rffi_platform.Has' entries, and if the CConfig tries to #include non-existing headers, then it is not a fatal error but simply an indication that the platform does not have the functions or names in question. Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/tool/rffi_platform.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/tool/rffi_platform.py Tue Sep 4 13:39:50 2007 @@ -127,40 +127,43 @@ if isinstance(value, CConfigEntry): entries.append((key, value)) - writer = _CWriter(CConfig) - writer.write_header() - for key, entry in entries: - writer.write_entry(key, entry) - - f = writer.f - writer.start_main() - for key, entry in entries: - writer.write_entry_main(key) - writer.close() - - include_dirs = getattr(CConfig, '_include_dirs_', []) - infolist = list(run_example_code(writer.path, include_dirs)) - assert len(infolist) == len(entries) - - resultinfo = {} - resultentries = {} - for info, (key, entry) in zip(infolist, entries): - resultinfo[key] = info - resultentries[entry] = key - - result = ConfigResult(CConfig, resultinfo, resultentries) - for name, entry in entries: - result.get_entry_result(entry) + if entries: # can be empty if there are only CConfigSingleEntries + writer = _CWriter(CConfig) + writer.write_header() + for key, entry in entries: + writer.write_entry(key, entry) + + f = writer.f + writer.start_main() + for key, entry in entries: + writer.write_entry_main(key) + writer.close() + + include_dirs = getattr(CConfig, '_include_dirs_', []) + infolist = list(run_example_code(writer.path, include_dirs)) + assert len(infolist) == len(entries) + + resultinfo = {} + resultentries = {} + for info, (key, entry) in zip(infolist, entries): + resultinfo[key] = info + resultentries[entry] = key + + result = ConfigResult(CConfig, resultinfo, resultentries) + for name, entry in entries: + result.get_entry_result(entry) + res = result.get_result() + else: + res = {} for key in dir(CConfig): value = getattr(CConfig, key) if isinstance(value, CConfigSingleEntry): writer = _CWriter(CConfig) writer.write_header() - result.result[value] = value.question(writer.ask_gcc) - result.entries[value] = key - - return result.get_result() + res[key] = value.question(writer.ask_gcc) + + return res # ____________________________________________________________ Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/branch/pypy-more-rtti-inprogress/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/branch/pypy-more-rtti-inprogress/rpython/tool/test/test_rffi_platform.py Tue Sep 4 13:39:50 2007 @@ -189,3 +189,5 @@ def test_has(): assert rffi_platform.has("x", "int x = 3;") assert not rffi_platform.has("x", "") + # has() should also not crash if it is given an invalid #include + assert not rffi_platform.has("x", "#include ") From antocuni at codespeak.net Tue Sep 4 14:28:55 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 4 Sep 2007 14:28:55 +0200 (CEST) Subject: [pypy-svn] r46304 - pypy/dist/pypy/translator/jvm Message-ID: <20070904122855.853DD8177@code0.codespeak.net> Author: antocuni Date: Tue Sep 4 14:28:54 2007 New Revision: 46304 Modified: pypy/dist/pypy/translator/jvm/constant.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/typesystem.py Log: these are needed for pypy-jvm to compile Modified: pypy/dist/pypy/translator/jvm/constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/constant.py (original) +++ pypy/dist/pypy/translator/jvm/constant.py Tue Sep 4 14:28:54 2007 @@ -95,6 +95,8 @@ self.hash_jcls = self.db.record_delegate_standalone_func_impl( self.value._dict.key_hash.graph) + CustomDictConst.record_dependencies(self) + def create_pointer(self, gen): gen.new_with_jtype(self.eq_jcls) gen.new_with_jtype(self.hash_jcls) Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Tue Sep 4 14:28:54 2007 @@ -10,7 +10,7 @@ jObject, jByteArray, jPyPyExcWrap, jIntegerClass, jLongClass, \ jDoubleClass, jCharClass, jStringBuilder, JvmScalarType, jArrayList, \ jObjectArray, jPyPyInterlink, jPyPyCustomDict, jPyPyEquals, \ - jPyPyHashCode, jMap, jWeakRef + jPyPyHashCode, jMap, jWeakRef, jSystem # ___________________________________________________________________________ # Miscellaneous helper functions @@ -359,6 +359,7 @@ OBJHASHCODE = Method.v(jObject, 'hashCode', (), jInt) OBJTOSTRING = Method.v(jObject, 'toString', (), jString) OBJEQUALS = Method.v(jObject, 'equals', (jObject,), jBool) +SYSTEMGC = Method.s(jSystem, 'gc', (), jVoid) INTTOSTRINGI = Method.s(jIntegerClass, 'toString', (jInt,), jString) LONGTOSTRINGL = Method.s(jLongClass, 'toString', (jLong,), jString) DOUBLETOSTRINGD = Method.s(jDoubleClass, 'toString', (jDouble,), jString) @@ -1070,7 +1071,8 @@ def push_null(self, OOTYPE): self.emit(ACONST_NULL) - DEFINED_INT_SYMBOLICS = {'MALLOC_ZERO_FILLED':1} + DEFINED_INT_SYMBOLICS = {'MALLOC_ZERO_FILLED':1, + '0 /* we are not jitted here */': 0} def push_primitive_constant(self, TYPE, value): if TYPE is ootype.Void: Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Tue Sep 4 14:28:54 2007 @@ -67,7 +67,7 @@ 'cast_ptr_to_weakadr': [CastPtrToWeakAddress], 'cast_weakadr_to_ptr': CastWeakAddressToPtr, - #'gc__collect': 'call void class [mscorlib]System.GC::Collect()', + 'gc__collect': jvmgen.SYSTEMGC, 'resume_point': Ignore, 'debug_assert': [], # TODO: implement? Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Tue Sep 4 14:28:54 2007 @@ -165,6 +165,7 @@ jIterator = JvmClassType('java.util.Iterator') jClass = JvmClassType('java.lang.Class') jStringBuilder = JvmClassType('java.lang.StringBuilder') +jSystem = JvmClassType('java.lang.System') jPrintStream = JvmClassType('java.io.PrintStream') jMath = JvmClassType('java.lang.Math') jList = JvmInterfaceType('java.util.List') From antocuni at codespeak.net Tue Sep 4 19:21:29 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 4 Sep 2007 19:21:29 +0200 (CEST) Subject: [pypy-svn] r46307 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070904172129.AE4D2812D@code0.codespeak.net> Author: antocuni Date: Tue Sep 4 19:21:28 2007 New Revision: 46307 Modified: pypy/dist/pypy/translator/jvm/test/test_constant.py Log: a failing test: JVM does not like classes to be too large, so we need to think a way to split the Constant class into multiple ones. Modified: pypy/dist/pypy/translator/jvm/test/test_constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_constant.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_constant.py Tue Sep 4 19:21:28 2007 @@ -129,3 +129,14 @@ def fn(x, y): return c1[x] + c2[y] assert self.interpret(fn, [0, 0]) == 3 + + def test_many_constants(self): + py.test.skip('fixme!') + N = 7500 + class A: + pass + mylist = [A() for i in range(N)] + def fn(x): + return mylist[x] + res = self.interpret(fn, [0]) + assert self.class_name(res) == 'A' From simonb at codespeak.net Tue Sep 4 20:21:20 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Tue, 4 Sep 2007 20:21:20 +0200 (CEST) Subject: [pypy-svn] r46308 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070904182120.19A2C812C@code0.codespeak.net> Author: simonb Date: Tue Sep 4 20:21:18 2007 New Revision: 46308 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: a lot of hacking to get array types to point to a 'base' array. i dont much like this, but checking it in now in case i change my mind later on Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Tue Sep 4 20:21:18 2007 @@ -31,9 +31,12 @@ self.typecode = typecode self.ndim = ndim self.s_base = s_base # we are a view into this + if s_base is not None: + assert s_base.s_base is None # shallow bases def get_base_annotation(self): return self.s_base or self + # can't set s_base to self because this screws up __eq__ def can_be_none(self): return True @@ -56,11 +59,11 @@ return s def method_transpose(self): - return SomeArray(self.typecode, self.ndim) + return SomeArray(self.typecode, self.ndim, self) class __extend__(pairtype(SomeArray, SomeArray)): - def union((s_arr1, s_arr2)): + def add((s_arr1, s_arr2)): item1 = s_arr1.get_item_type() item2 = s_arr2.get_item_type() typecode = None @@ -73,9 +76,9 @@ break if typecode is None: raise AnnotatorError() - return SomeArray(typecode) + return SomeArray(typecode, s_arr1.ndim) - add = sub = mul = div = truediv = union + sub = mul = div = add class __extend__(pairtype(SomeArray, SomeTuple)): @@ -152,6 +155,7 @@ _about_ = numpy.array def compute_result_annotation(self, s_list, s_dtype=None): + s_base = None if isinstance(s_list, SomeList): # First guess type from input list listitem = s_list.listdef.listitem @@ -161,6 +165,7 @@ elif isinstance(s_list, SomeArray): typecode = s_list.typecode ndim = s_list.ndim + s_base = s_list else: raise AnnotatorError("cannot build array from %s"%s_list) @@ -172,7 +177,12 @@ raise AnnotatorError("dtype is not a valid type specification") if typecode is None or typecode not in valid_typecodes: raise AnnotatorError("List item type not supported") - return SomeArray(typecode, ndim) + + if isinstance(s_list, SomeList): + # make phantom array annotation + s_base = SomeArray(typecode, ndim) + + return SomeArray(typecode, ndim, s_base) def specialize_call(self, hop, i_dtype=None): r_array = hop.r_result Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Tue Sep 4 20:21:18 2007 @@ -20,8 +20,8 @@ def gen_build_from_shape(ndim): unrolling_dims = unrolling_iterable(reversed(range(ndim))) - def ll_build_from_shape(ARRAY, shape): - array = ll_allocate(ARRAY, ndim) + def ll_build_from_shape(ARRAY, BASE, shape): + array = ll_allocate(ARRAY, BASE, ndim) itemsize = 1 for i in unrolling_dims: attr = 'item%d'%i @@ -58,7 +58,7 @@ ("strides", INDEXARRAY), ("backstrides", INDEXARRAY), #("factors", INDEXARRAY), - ("ao", ARRAY), +# ("ao", ARRAY), # EECK!! this makes too many iter types !! ("dataptr", ARRAY.TO.dataptr), # pointer to current item #("contiguous", Bool), )) @@ -75,17 +75,19 @@ unroll_ndim = unrolling_iterable(range(ndim)) unroll_ndim_rev = unrolling_iterable(reversed(range(ndim))) - def ll_iter_reset(it): + def ll_iter_reset(it, dataptr): it.index = 0 - it.dataptr = it.ao.dataptr +# it.dataptr = it.ao.dataptr + it.dataptr = dataptr for i in unroll_ndim: it.coordinates[i] = 0 ll_iter_reset._always_inline_ = True def ll_iter_new(ITER, ao, iter_reset=ll_iter_reset): assert ao.dataptr + assert ao.ndim == ndim it = malloc(ITER) - it.ao = ao +# it.ao = ao it.nd_m1 = ndim - 1 it.size = ll_mul_list(ao.shape, ndim) #it.factors[nd-1] = 1 @@ -95,7 +97,7 @@ it.backstrides[i] = it.strides[i] * it.dims_m1[i] #if i > 0: #it.factors[nd-i-1] = it.factors[nd]*ao.shape[nd-i] - iter_reset(it) + iter_reset(it, ao.dataptr) return it ll_iter_new._always_inline_ = True @@ -108,7 +110,7 @@ raise Exception("array is not broadcastable to correct shape") # XX raise here ? j += 1 it = malloc(ITER) - it.ao = ao +# it.ao = ao it.size = ll_mul_list(ao.shape, ndim) it.nd_m1 = ndim - 1 #it.factors[nd-1] = 1 @@ -123,7 +125,7 @@ it.backstrides[i] = it.strides[i] * it.dims_m1[i] #if i > 0: #it.factors[nd-i-1] = it.factors[nd-i]*shape[nd-i] - iter_reset(it) + iter_reset(it, ao.dataptr) return it ll_iter_broadcast_to_shape._always_inline_ = True @@ -147,60 +149,84 @@ def ll_binary_op(p0, p1, p2, op=lambda x,y:x+y): p0[0] = op(p1[0], p2[0]) -def ll_array_unary_op(iter_new0, iter_next0, ITER0, array0, - iter_new1, iter_next1, ITER1, array1,): -# op=ll_unary_op): - it0 = iter_new0(ITER0, array0) - it1 = iter_new1(ITER1, array1) +def ll_array_unary_op(iter_new, iter_next, ITER, array0, array1): + it0 = iter_new(ITER, array0) + it1 = iter_new(ITER, array1) while it0.index < it0.size: ll_unary_op(it0.dataptr, it1.dataptr) - iter_next0(it0) - iter_next1(it1) + iter_next(it0) + iter_next(it1) + +def dim_of_ARRAY(ARRAY): + return ARRAY.TO.shape.length + +def BASE_TYPE(ARRAY): + return ARRAY.TO.base.TO class ArrayRepr(Repr): def __init__(self, rtyper, s_array): self.s_array = s_array self.s_value = s_array.get_item_type() self.ndim = s_array.ndim - self.item_repr = rtyper.getrepr(self.s_value) + self.item_repr = rtyper.getrepr(self.s_value) # XX rename r_item XX self.ITEM = self.item_repr.lowleveltype ITEMARRAY = GcArray(self.ITEM, hints={'nolength':True}) self.INDEXARRAY = FixedSizeArray(NPY_INTP, self.ndim) self.itemsize = sizeof(self.ITEM) - FORWARD = GcForwardReference() + #BASE = GcForwardReference() + BASE = Void # whatever... + s_base = s_array.s_base + r_base = None + if s_base is None: + # we are pointing to ourself + BASE = Ptr(GcForwardReference()) + else: + #print s_base, s_array + assert rtyper.makekey(s_base) != rtyper.makekey(s_array) + r_base = rtyper.getrepr(s_base) + BASE = Ptr(r_base.STRUCT) DATA_PTR = Ptr(FixedSizeArray(self.ITEM, 1)) - STRUCT = GcStruct("array", + self.STRUCT = GcStruct("array", ("data", Ptr(ITEMARRAY)), # pointer to raw data buffer ("dataptr", DATA_PTR), # pointer to first element ("ndim", Signed), # number of dimensions ("shape", self.INDEXARRAY), # size in each dimension ("strides", self.INDEXARRAY), # elements to jump to get to the # next element in each dimension - ("base", Ptr(FORWARD)), # we are a view into this array + ("base", BASE), # we are a view into this array ) - self.ARRAY = Ptr(STRUCT) - STRUCT.base.TO.become(STRUCT) + self.ARRAY = Ptr(self.STRUCT) + if s_base is None: +# self.ARRAY.TO.base.TO.become(self.STRUCT) + BASE.TO.become(self.STRUCT) + self.BASE = BASE + self.r_base = r_base self.lowleveltype = self.ARRAY self.ITER = ARRAY_ITER(self.ARRAY, self.INDEXARRAY) def build_from_array(self, llops, v_array): + #self.set_base_repr(self) cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) - return llops.gendirectcall(ll_build_alias, cARRAY, v_array) + cBASE = inputconst(lltype.Void, self.BASE.TO) + return llops.gendirectcall(ll_build_alias, cARRAY, cBASE, v_array) def build_from_shape(self, llops, r_tuple, v_tuple): + #self.set_base_repr(self) cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) cTUPLE = inputconst(lltype.Void, r_tuple.lowleveltype.TO) ndim = self.s_array.ndim ll_build_from_shape = gen_build_from_shape(ndim) c_ndim = inputconst(lltype.Signed, ndim) assert ndim == len(r_tuple.items_r) - rval = llops.gendirectcall(ll_build_from_shape, cARRAY, v_tuple) + cBASE = inputconst(lltype.Void, self.BASE.TO) + rval = llops.gendirectcall(ll_build_from_shape, cARRAY, cBASE, v_tuple) return rval def rtype_method_transpose(self, hop): [v_self] = hop.inputargs(self) cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) - return hop.gendirectcall(ll_transpose, cARRAY, v_self) + cBASE = inputconst(lltype.Void, self.BASE.TO) + return hop.gendirectcall(ll_transpose, cARRAY, cBASE, v_self) def get_ndim(self, hop, v_array): cname = inputconst(Void, 'ndim') @@ -235,14 +261,18 @@ return ArrayRepr(rtyper, self) def rtyper_makekey(self): - return self.__class__, self.typecode, self.ndim + key = self.__class__, self.typecode, self.ndim + if self.s_base is not None: + key = key + self.s_base.rtyper_makekey() + return key class __extend__(pairtype(ArrayRepr, ArrayRepr)): def rtype_add((r_arr1, r_arr2), hop): v_arr1, v_arr2 = hop.inputargs(r_arr1, r_arr2) cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) - return hop.gendirectcall(ll_add, cARRAY, v_arr1, v_arr2) + cBASE = foo + return hop.gendirectcall(ll_add, cARRAY, cBASE, v_arr1, v_arr2) #class __extend__(pairtype(ArrayRepr, Repr)): # <------ USE THIS ?? @@ -255,17 +285,28 @@ v_array, v_index = hop.inputargs(r_arr, Signed) return hop.gendirectcall(ll_getitem1, v_array, v_index) +class __extend__(pairtype(ArrayRepr, AbstractSliceRepr)): + def rtype_setitem((r_arr, r_slc), hop): + r_item = hop.args_r[2] + v_array, v_slc, v_item = hop.inputargs(r_arr, r_slc, r_item) + cITER = hop.inputconst(Void, r_arr.ITER.TO) + iter_new, iter_next = gen_iter_funcs(r_arr.ndim) + cnew = hop.inputconst(Void, iter_new) + cnext = hop.inputconst(Void, iter_next) + assert r_arr.ndim == r_item.ndim + return hop.gendirectcall(ll_array_unary_op, cnew, cnext, cITER, v_array, v_item) + def gen_getset_item(ndim): unrolling_dims = unrolling_iterable(range(ndim)) - def ll_get_item(ARRAY, ao, tpl): - array = ll_allocate(ARRAY, ndim) + def ll_get_item(ARRAY, BASE, ao, tpl): + array = ll_allocate(ARRAY, BASE, ndim) idx = 0 for i in unrolling_dims: idx += ao.strides[i] * getattr(tpl, 'item%d'%i) return ao.data[idx] - def ll_set_item(ARRAY, ao, tpl, value): - array = ll_allocate(ARRAY, ndim) + def ll_set_item(ARRAY, BASE, ao, tpl, value): + array = ll_allocate(ARRAY, BASE, ndim) idx = 0 for i in unrolling_dims: idx += ao.strides[i] * getattr(tpl, 'item%d'%i) @@ -273,71 +314,115 @@ return ll_get_item, ll_set_item +def get_view_ndim(r_tpl): + return len([r_item for r_item in r_tpl.items_r if isinstance(r_item, AbstractSliceRepr)]) + +def gen_get_view(r_tpl): + ndim = get_view_ndim(r_tpl) + unroll_r_tpl = unrolling_iterable(enumerate(r_tpl.items_r)) + def ll_get_view(ARRAY, BASE, ao, tpl): + array = ll_allocate(ARRAY, BASE, ndim) + array.base = ao + dataptr = direct_arrayitems(ao.data) + src_i = 0 + tgt_i = 0 + for src_i, r_item in unroll_r_tpl: + if isinstance(r_item, IntegerRepr): + r_int = r_item + dataptr = direct_ptradd(dataptr, getattr(tpl, 'item%d'%src_i)) + else: + r_slice = r_item + array.shape[tgt_i] = ao.shape[src_i] + array.strides[tgt_i] = ao.strides[tgt_i] + tgt_i += 1 + assert tgt_i == ndim + array.dataptr = dataptr + return array + return ll_get_view + + class __extend__(pairtype(ArrayRepr, AbstractTupleRepr)): def rtype_getitem((r_arr, r_tpl), hop): v_array, v_tuple = hop.inputargs(r_arr, r_tpl) - cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) - get_item, set_item = gen_getset_item(r_arr.ndim) - return hop.gendirectcall(get_item, cARRAY, v_array, v_tuple) + ndim = get_view_ndim(r_tpl) + if ndim == 0: + # return a scalar + cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) + cBASE = inputconst(lltype.Void, r_arr.BASE.TO) + get_item, set_item = gen_getset_item(r_arr.ndim) + return hop.gendirectcall(get_item, cARRAY, cBASE, v_array, v_tuple) + r_result = hop.r_result + ARRAY = r_result.ARRAY + assert dim_of_ARRAY(ARRAY) == ndim + cARRAY = hop.inputconst(Void, ARRAY.TO) + cBASE = inputconst(lltype.Void, r_result.BASE.TO) + ll_get_view = gen_get_view(r_tpl) + return hop.gendirectcall(ll_get_view, cARRAY, cBASE, v_array, v_tuple) def rtype_setitem((r_arr, r_tpl), hop): - v_array, v_tuple, v_item = hop.inputargs(r_arr, r_tpl, hop.args_r[2]) - cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) - get_item, set_item = gen_getset_item(r_arr.ndim) - return hop.gendirectcall(set_item, cARRAY, v_array, v_tuple, v_item) - -class __extend__(pairtype(ArrayRepr, AbstractSliceRepr)): - def rtype_setitem((r_arr, r_slc), hop): r_item = hop.args_r[2] - v_array, v_slc, v_item = hop.inputargs(r_arr, r_slc, r_item) - cITER0 = hop.inputconst(Void, r_arr.ITER.TO) - cITER1 = hop.inputconst(Void, r_item.ITER.TO) - iter_new0, iter_next0 = gen_iter_funcs(r_arr.ndim) - iter_new1, iter_next1 = gen_iter_funcs(r_item.ndim) - cnew0 = hop.inputconst(Void, iter_new0) - cnext0 = hop.inputconst(Void, iter_next0) - cnew1 = hop.inputconst(Void, iter_new1) - cnext1 = hop.inputconst(Void, iter_next1) - return hop.gendirectcall(ll_array_unary_op, - cnew0, cnext0, cITER0, v_array, cnew1, cnext1, cITER1, v_item) - + v_array, v_tuple, v_item = hop.inputargs(r_arr, r_tpl, r_item) + ndim = get_view_ndim(r_tpl) + if isinstance(r_item, ArrayRepr): + get_view = gen_get_view(r_tpl) + cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) + cBASE = inputconst(lltype.Void, r_arr.BASE.TO) + v_view = hop.gendirectcall(get_view, cARRAY, cBASE, v_array, v_tuple) + iter_new, iter_next = gen_iter_funcs(r_arr.ndim) + assert ndim == r_item.ndim + cnew = hop.inputconst(Void, iter_new) + cnext = hop.inputconst(Void, iter_next) + cITER = hop.inputconst(Void, r_item.ITER.TO) + return hop.gendirectcall(ll_array_unary_op, cnew, cnext, cITER, v_array, v_view) + else: + # Set from scalar + assert ndim == 0 + cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) + cBASE = inputconst(lltype.Void, r_arr.BASE.TO) + get_item, set_item = gen_getset_item(r_arr.ndim) + return hop.gendirectcall(set_item, cARRAY, cBASE, v_array, v_tuple, v_item) + -#class __extend__(pairtype(ArrayRepr, AbstractSliceRepr)): -# # promote and delegate XX doesn't work -# def rtype_setitem((r_arr, r_slc), hop): -# r_tpl = TupleRepr(hop.rtyper, [r_slc]) -# return pair(r_arr, r_tpl).rtype_setitem(hop) -# -# def rtype_getitem((r_arr, r_slc), hop): -# r_tpl = TupleRepr(hop.rtyper, [r_slc]) -# return pair(r_arr, r_tpl).rtype_getitem(hop) +class __extend__(pairtype(ArrayRepr, ArrayRepr)): + def convert_from_to((r_arr0, r_arr1), v, llops): + print "%r -> %r"%( + (r_arr0, r_arr0.r_base), + (r_arr1, r_arr1.r_base), + ) + assert 0 class __extend__(pairtype(AbstractBaseListRepr, ArrayRepr)): def convert_from_to((r_lst, r_arr), v, llops): + #import py;py.test.skip() + assert 0 if r_lst.listitem is None: return NotImplemented if r_lst.item_repr != r_arr.item_repr: assert 0, (r_lst, r_arr.item_repr) return NotImplemented cARRAY = inputconst(lltype.Void, r_arr.lowleveltype.TO) - return llops.gendirectcall(ll_build_from_list, cARRAY, v) + cBASE = inputconst(lltype.Void, r_arr.BASE.TO) + return llops.gendirectcall(ll_build_from_list, cARRAY, cBASE, v) class __extend__(pairtype(AbstractRangeRepr, ArrayRepr)): def convert_from_to((r_rng, r_arr), v, llops): + #import py;py.test.skip() + assert 0 cARRAY = inputconst(lltype.Void, r_arr.lowleveltype.TO) - return llops.gendirectcall(ll_build_from_list, cARRAY, v) + cBASE = inputconst(lltype.Void, r_arr.BASE.TO) + return llops.gendirectcall(ll_build_from_list, cARRAY, cBASE, v) -def ll_allocate(ARRAY, ndim): +def ll_allocate(ARRAY, BASE, ndim): array = malloc(ARRAY) array.ndim = ndim - array.base = nullptr(ARRAY) + array.base = nullptr(BASE) array.data = nullptr(ARRAY.data.TO) array.dataptr = nullptr(ARRAY.dataptr.TO) return array -def ll_build_from_list(ARRAY, lst): +def ll_build_from_list(ARRAY, BASE, lst): size = lst.ll_length() - array = ll_allocate(ARRAY, 1) + array = ll_allocate(ARRAY, BASE, 1) array.shape[0] = size array.strides[0] = 1 array.data = malloc(ARRAY.data.TO, size) @@ -348,8 +433,8 @@ array.dataptr = direct_arrayitems(array.data) return array -def ll_build_alias(ARRAY, ao): - array = ll_allocate(ARRAY, ao.ndim) +def ll_build_alias(ARRAY, BASE, ao): + array = ll_allocate(ARRAY, BASE, ao.ndim) array.data = ao.data # alias data array.base = ao if ao.base: @@ -379,8 +464,8 @@ array.dataptr = direct_arrayitems(array.data) return array -def ll_transpose(ARRAY, a1): - a2 = ll_build_alias(ARRAY, a1) +def ll_transpose(ARRAY, BASE, a1): + a2 = ll_build_alias(ARRAY, BASE, a1) # XX do something to a2 return a2 Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Tue Sep 4 20:21:18 2007 @@ -22,6 +22,17 @@ test_c_compile = True test_llvm_compile = False +def fromlist1(lst, dtype=''): + if dtype: + a = numpy.empty((len(lst),), dtype=dtype) + else: + a = numpy.empty((len(lst),)) + idx = 0 + while idx < len(lst): + a[idx] = lst[idx] + idx += 1 + return a + def access_array(item): my_array = numpy.array([item]) return my_array[0] @@ -98,20 +109,22 @@ assert s.ndim == 2 def test_annotate_array_add(self): + py.test.skip() def f(): - a1 = numpy.array([1,2]) - a2 = numpy.array([6,9]) + a1 = fromlist1([1,2]) + a2 = fromlist1([6,9]) return a1 + a2 t = TranslationContext() a = t.buildannotator() s = a.build_types(f, []) - assert s.typecode == 'i' + #assert s.typecode == 'i' def test_annotate_array_add_coerce(self): + py.test.skip() def f(): - a1 = numpy.array([1,2]) - a2 = numpy.array([6.,9.]) + a1 = fromlist1([1,2]) + a2 = fromlist1([6.,9.]) return a1 + a2 t = TranslationContext() @@ -121,7 +134,7 @@ def test_annotate_array_dtype(self): def f(): - a1 = numpy.array([1,2], dtype='d') + a1 = fromlist1([1,2], dtype='d') return a1 t = TranslationContext() @@ -131,8 +144,8 @@ def test_annotate_array_array(self): def f(): - a1 = numpy.array([1,2], dtype='d') - a2 = numpy.array(a1) + a1 = fromlist1([1,2], dtype='d') + a2 = fromlist1(a1) return a2 t = TranslationContext() @@ -142,7 +155,7 @@ def test_annotate_array_attr(self): def f(): - a1 = numpy.array([1,2]) + a1 = fromlist1([1,2]) return a1.shape t = TranslationContext() @@ -152,7 +165,7 @@ def test_annotate_array_method(self): def f(): - a1 = numpy.array([1,2]) + a1 = fromlist1([1,2]) return a1.transpose() t = TranslationContext() @@ -160,6 +173,21 @@ s = a.build_types(f, []) assert type(s) == SomeArray + def test_annotate_indexing(self): + def f(): + a = numpy.empty((4,3), dtype='i') + c = a[:,0] + return c + t = TranslationContext() + a = t.buildannotator() + s_array = a.build_types(f, []) + assert type(s_array) == SomeArray + assert type(s_array.s_base) == SomeArray + assert s_array.ndim == 1 + assert s_array.s_base.ndim == 2 + assert s_array.s_base.s_base == None + + from pypy.objspace.flow.model import checkgraph, flatten, Block, mkentrymap from pypy.translator.backendopt.malloc import LLTypeMallocRemover @@ -177,8 +205,13 @@ def test_specialize_array_create(self): def f(): - a = numpy.array([1,20]) - b = numpy.array(a) +# a = numpy.empty((2,)) +# a[0] = 1 +# a[1] = 20 +# return a + a = fromlist1([1,20]) +# b = numpy.array(a) # XX + b = a return b res = interpret(f, []) @@ -195,7 +228,7 @@ def test_specialize_array_access(self): def access_with_variable(): - my_array = numpy.array(range(10), dtype='i') + my_array = fromlist1(range(10), dtype='i') my_array[2] = 2 sum = 0 for idx in range(10): @@ -207,9 +240,10 @@ assert res == 45 def test_specialize_array_add(self): + py.test.skip() def f(): - a1 = numpy.array([1.,2.]) - a2 = numpy.array([6,9]) + a1 = fromlist1([1.,2.]) + a2 = fromlist1([6,9]) return a1 + a2 res = interpret(f, []) @@ -218,7 +252,7 @@ def test_specialize_array_attr(self): def f(): - a = numpy.array([1,2]) + a = fromlist1([1,2]) return a.ndim res = interpret(f, []) @@ -226,11 +260,9 @@ def test_specialize_base(self): def f(): - a = numpy.array([1,2]) - b = numpy.array(a) - # Aha: a.base is a phantom array we made - # when converting from a list. - return b.base is a.base + a = numpy.empty((2,)) + b = a[:,] + return b.base is a res = interpret(f, []) assert res @@ -259,23 +291,27 @@ def test_specialize_array_method(self): def f(): - a = numpy.array([1,2]) + a = numpy.empty((2,)) return a.transpose() res = interpret(f, []) - assert res.data[0] == 1 - assert res.data[1] == 2 +# assert res.data[0] == 1 +# assert res.data[1] == 2 def test_specialize_indexing(self): def f(): - a = numpy.empty((3,), dtype='i') - b = numpy.array([5,55,555]) - a[:] = b - return a + a = numpy.empty((4,3), dtype='i') + a[0,0] = 5 + a[1,0] = 55 + a[2,0] = 555 + c = a[:,0] + return c res = interpret(f, []) - assert res.data[0] == 5 - assert res.data[1] == 55 - assert res.data[2] == 555 + assert res.dataptr[0] == 5 + assert res.dataptr[3] == 55 + assert res.dataptr[6] == 555 + assert res.shape.item0 == 4 + assert res.strides.item0 == 3 def test_specialize_multi(self): def f(ii, jj): @@ -288,9 +324,10 @@ assert interpret(f, [3, 4]) == 12 def test_malloc_remove(self): + py.test.skip('this test requires _always_inline_ magic hook') def f(): a = numpy.empty((3,), dtype='i') - b = numpy.array([5,55,555]) + b = fromlist1([5,55,555], dtype='i') a[:] = b return a t = TranslationContext() @@ -319,7 +356,7 @@ def test_compile_array_access(self): def access_array(index): a = numpy.empty((3,), dtype='i') - b = numpy.array([5,55,555]) + b = fromlist1([5,55,555], dtype='i') a[:] = b a[0] = 1 return a[index] From simonb at codespeak.net Tue Sep 4 21:09:39 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Tue, 4 Sep 2007 21:09:39 +0200 (CEST) Subject: [pypy-svn] r46309 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070904190939.EE69F8114@code0.codespeak.net> Author: simonb Date: Tue Sep 4 21:09:39 2007 New Revision: 46309 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: get rid of base attribute. can use data attribute to detect aliases Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Tue Sep 4 21:09:39 2007 @@ -25,18 +25,11 @@ #'f' : SomeFloat(), # XX single precision float XX 'd' : SomeFloat(), } - def __init__(self, typecode, ndim=1, s_base=None): + def __init__(self, typecode, ndim=1): if not typecode in self.typecode_to_item: raise AnnotatorError("bad typecode: %r"%typecode) self.typecode = typecode self.ndim = ndim - self.s_base = s_base # we are a view into this - if s_base is not None: - assert s_base.s_base is None # shallow bases - - def get_base_annotation(self): - return self.s_base or self - # can't set s_base to self because this screws up __eq__ def can_be_none(self): return True @@ -52,14 +45,12 @@ s = SomeTuple([SomeInteger()]*s_array.ndim) elif attr == 'ndim': s = SomeInteger() - elif attr == 'base': - s = s_array.get_base_annotation() if s is None: return SomeObject.getattr(s_array, s_attr) return s def method_transpose(self): - return SomeArray(self.typecode, self.ndim, self) + return SomeArray(self.typecode, self.ndim) class __extend__(pairtype(SomeArray, SomeArray)): @@ -111,7 +102,7 @@ if s_array.ndim == 0 and len(s_index.items): raise AnnotatorError("indexing rank zero array with nonempty tuple") if ndim > 0: - return SomeArray(s_array.typecode, ndim, s_array.get_base_annotation()) + return SomeArray(s_array.typecode, ndim) return s_array.get_item_type() # These two up-cast the index to SomeTuple and call above. @@ -155,7 +146,6 @@ _about_ = numpy.array def compute_result_annotation(self, s_list, s_dtype=None): - s_base = None if isinstance(s_list, SomeList): # First guess type from input list listitem = s_list.listdef.listitem @@ -165,7 +155,6 @@ elif isinstance(s_list, SomeArray): typecode = s_list.typecode ndim = s_list.ndim - s_base = s_list else: raise AnnotatorError("cannot build array from %s"%s_list) @@ -178,11 +167,7 @@ if typecode is None or typecode not in valid_typecodes: raise AnnotatorError("List item type not supported") - if isinstance(s_list, SomeList): - # make phantom array annotation - s_base = SomeArray(typecode, ndim) - - return SomeArray(typecode, ndim, s_base) + return SomeArray(typecode, ndim) def specialize_call(self, hop, i_dtype=None): r_array = hop.r_result Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Tue Sep 4 21:09:39 2007 @@ -20,8 +20,8 @@ def gen_build_from_shape(ndim): unrolling_dims = unrolling_iterable(reversed(range(ndim))) - def ll_build_from_shape(ARRAY, BASE, shape): - array = ll_allocate(ARRAY, BASE, ndim) + def ll_build_from_shape(ARRAY, shape): + array = ll_allocate(ARRAY, ndim) itemsize = 1 for i in unrolling_dims: attr = 'item%d'%i @@ -77,7 +77,6 @@ def ll_iter_reset(it, dataptr): it.index = 0 -# it.dataptr = it.ao.dataptr it.dataptr = dataptr for i in unroll_ndim: it.coordinates[i] = 0 @@ -87,7 +86,6 @@ assert ao.dataptr assert ao.ndim == ndim it = malloc(ITER) -# it.ao = ao it.nd_m1 = ndim - 1 it.size = ll_mul_list(ao.shape, ndim) #it.factors[nd-1] = 1 @@ -110,7 +108,6 @@ raise Exception("array is not broadcastable to correct shape") # XX raise here ? j += 1 it = malloc(ITER) -# it.ao = ao it.size = ll_mul_list(ao.shape, ndim) it.nd_m1 = ndim - 1 #it.factors[nd-1] = 1 @@ -160,9 +157,6 @@ def dim_of_ARRAY(ARRAY): return ARRAY.TO.shape.length -def BASE_TYPE(ARRAY): - return ARRAY.TO.base.TO - class ArrayRepr(Repr): def __init__(self, rtyper, s_array): self.s_array = s_array @@ -173,18 +167,6 @@ ITEMARRAY = GcArray(self.ITEM, hints={'nolength':True}) self.INDEXARRAY = FixedSizeArray(NPY_INTP, self.ndim) self.itemsize = sizeof(self.ITEM) - #BASE = GcForwardReference() - BASE = Void # whatever... - s_base = s_array.s_base - r_base = None - if s_base is None: - # we are pointing to ourself - BASE = Ptr(GcForwardReference()) - else: - #print s_base, s_array - assert rtyper.makekey(s_base) != rtyper.makekey(s_array) - r_base = rtyper.getrepr(s_base) - BASE = Ptr(r_base.STRUCT) DATA_PTR = Ptr(FixedSizeArray(self.ITEM, 1)) self.STRUCT = GcStruct("array", ("data", Ptr(ITEMARRAY)), # pointer to raw data buffer @@ -193,50 +175,34 @@ ("shape", self.INDEXARRAY), # size in each dimension ("strides", self.INDEXARRAY), # elements to jump to get to the # next element in each dimension - ("base", BASE), # we are a view into this array ) self.ARRAY = Ptr(self.STRUCT) - if s_base is None: -# self.ARRAY.TO.base.TO.become(self.STRUCT) - BASE.TO.become(self.STRUCT) - self.BASE = BASE - self.r_base = r_base self.lowleveltype = self.ARRAY self.ITER = ARRAY_ITER(self.ARRAY, self.INDEXARRAY) def build_from_array(self, llops, v_array): - #self.set_base_repr(self) cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) - cBASE = inputconst(lltype.Void, self.BASE.TO) - return llops.gendirectcall(ll_build_alias, cARRAY, cBASE, v_array) + return llops.gendirectcall(ll_build_alias, cARRAY, v_array) def build_from_shape(self, llops, r_tuple, v_tuple): - #self.set_base_repr(self) cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) cTUPLE = inputconst(lltype.Void, r_tuple.lowleveltype.TO) ndim = self.s_array.ndim ll_build_from_shape = gen_build_from_shape(ndim) c_ndim = inputconst(lltype.Signed, ndim) assert ndim == len(r_tuple.items_r) - cBASE = inputconst(lltype.Void, self.BASE.TO) - rval = llops.gendirectcall(ll_build_from_shape, cARRAY, cBASE, v_tuple) + rval = llops.gendirectcall(ll_build_from_shape, cARRAY, v_tuple) return rval def rtype_method_transpose(self, hop): [v_self] = hop.inputargs(self) cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) - cBASE = inputconst(lltype.Void, self.BASE.TO) - return hop.gendirectcall(ll_transpose, cARRAY, cBASE, v_self) + return hop.gendirectcall(ll_transpose, cARRAY, v_self) def get_ndim(self, hop, v_array): cname = inputconst(Void, 'ndim') return hop.llops.genop('getfield', [v_array, cname], resulttype=Signed) - def get_base(self, hop, v_array): - cname = inputconst(Void, 'base') - v_base = hop.llops.genop('getfield', [v_array, cname], resulttype=self.ARRAY) - return v_base - def get_shape(self, hop, v_array): TUPLE = TUPLE_TYPE([Signed]*self.ndim) cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) @@ -262,8 +228,6 @@ def rtyper_makekey(self): key = self.__class__, self.typecode, self.ndim - if self.s_base is not None: - key = key + self.s_base.rtyper_makekey() return key @@ -271,8 +235,7 @@ def rtype_add((r_arr1, r_arr2), hop): v_arr1, v_arr2 = hop.inputargs(r_arr1, r_arr2) cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) - cBASE = foo - return hop.gendirectcall(ll_add, cARRAY, cBASE, v_arr1, v_arr2) + return hop.gendirectcall(ll_add, cARRAY, v_arr1, v_arr2) #class __extend__(pairtype(ArrayRepr, Repr)): # <------ USE THIS ?? @@ -298,15 +261,15 @@ def gen_getset_item(ndim): unrolling_dims = unrolling_iterable(range(ndim)) - def ll_get_item(ARRAY, BASE, ao, tpl): - array = ll_allocate(ARRAY, BASE, ndim) + def ll_get_item(ARRAY, ao, tpl): + array = ll_allocate(ARRAY, ndim) idx = 0 for i in unrolling_dims: idx += ao.strides[i] * getattr(tpl, 'item%d'%i) return ao.data[idx] - def ll_set_item(ARRAY, BASE, ao, tpl, value): - array = ll_allocate(ARRAY, BASE, ndim) + def ll_set_item(ARRAY, ao, tpl, value): + array = ll_allocate(ARRAY, ndim) idx = 0 for i in unrolling_dims: idx += ao.strides[i] * getattr(tpl, 'item%d'%i) @@ -320,9 +283,8 @@ def gen_get_view(r_tpl): ndim = get_view_ndim(r_tpl) unroll_r_tpl = unrolling_iterable(enumerate(r_tpl.items_r)) - def ll_get_view(ARRAY, BASE, ao, tpl): - array = ll_allocate(ARRAY, BASE, ndim) - array.base = ao + def ll_get_view(ARRAY, ao, tpl): + array = ll_allocate(ARRAY, ndim) dataptr = direct_arrayitems(ao.data) src_i = 0 tgt_i = 0 @@ -337,6 +299,7 @@ tgt_i += 1 assert tgt_i == ndim array.dataptr = dataptr + array.data = ao.data # keep a ref return array return ll_get_view @@ -348,16 +311,14 @@ if ndim == 0: # return a scalar cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) - cBASE = inputconst(lltype.Void, r_arr.BASE.TO) get_item, set_item = gen_getset_item(r_arr.ndim) - return hop.gendirectcall(get_item, cARRAY, cBASE, v_array, v_tuple) + return hop.gendirectcall(get_item, cARRAY, v_array, v_tuple) r_result = hop.r_result ARRAY = r_result.ARRAY assert dim_of_ARRAY(ARRAY) == ndim cARRAY = hop.inputconst(Void, ARRAY.TO) - cBASE = inputconst(lltype.Void, r_result.BASE.TO) ll_get_view = gen_get_view(r_tpl) - return hop.gendirectcall(ll_get_view, cARRAY, cBASE, v_array, v_tuple) + return hop.gendirectcall(ll_get_view, cARRAY, v_array, v_tuple) def rtype_setitem((r_arr, r_tpl), hop): r_item = hop.args_r[2] @@ -366,8 +327,7 @@ if isinstance(r_item, ArrayRepr): get_view = gen_get_view(r_tpl) cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) - cBASE = inputconst(lltype.Void, r_arr.BASE.TO) - v_view = hop.gendirectcall(get_view, cARRAY, cBASE, v_array, v_tuple) + v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_tuple) iter_new, iter_next = gen_iter_funcs(r_arr.ndim) assert ndim == r_item.ndim cnew = hop.inputconst(Void, iter_new) @@ -378,16 +338,15 @@ # Set from scalar assert ndim == 0 cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) - cBASE = inputconst(lltype.Void, r_arr.BASE.TO) get_item, set_item = gen_getset_item(r_arr.ndim) - return hop.gendirectcall(set_item, cARRAY, cBASE, v_array, v_tuple, v_item) + return hop.gendirectcall(set_item, cARRAY, v_array, v_tuple, v_item) class __extend__(pairtype(ArrayRepr, ArrayRepr)): def convert_from_to((r_arr0, r_arr1), v, llops): print "%r -> %r"%( - (r_arr0, r_arr0.r_base), - (r_arr1, r_arr1.r_base), + (r_arr0, ), + (r_arr1, ), ) assert 0 @@ -401,28 +360,25 @@ assert 0, (r_lst, r_arr.item_repr) return NotImplemented cARRAY = inputconst(lltype.Void, r_arr.lowleveltype.TO) - cBASE = inputconst(lltype.Void, r_arr.BASE.TO) - return llops.gendirectcall(ll_build_from_list, cARRAY, cBASE, v) + return llops.gendirectcall(ll_build_from_list, cARRAY, v) class __extend__(pairtype(AbstractRangeRepr, ArrayRepr)): def convert_from_to((r_rng, r_arr), v, llops): #import py;py.test.skip() assert 0 cARRAY = inputconst(lltype.Void, r_arr.lowleveltype.TO) - cBASE = inputconst(lltype.Void, r_arr.BASE.TO) - return llops.gendirectcall(ll_build_from_list, cARRAY, cBASE, v) + return llops.gendirectcall(ll_build_from_list, cARRAY, v) -def ll_allocate(ARRAY, BASE, ndim): +def ll_allocate(ARRAY, ndim): array = malloc(ARRAY) array.ndim = ndim - array.base = nullptr(BASE) array.data = nullptr(ARRAY.data.TO) array.dataptr = nullptr(ARRAY.dataptr.TO) return array -def ll_build_from_list(ARRAY, BASE, lst): +def ll_build_from_list(ARRAY, lst): size = lst.ll_length() - array = ll_allocate(ARRAY, BASE, 1) + array = ll_allocate(ARRAY, 1) array.shape[0] = size array.strides[0] = 1 array.data = malloc(ARRAY.data.TO, size) @@ -433,12 +389,9 @@ array.dataptr = direct_arrayitems(array.data) return array -def ll_build_alias(ARRAY, BASE, ao): - array = ll_allocate(ARRAY, BASE, ao.ndim) +def ll_build_alias(ARRAY, ao): + array = ll_allocate(ARRAY, ao.ndim) array.data = ao.data # alias data - array.base = ao - if ao.base: - array.base = ao.base for i in range(ao.ndim): array.shape[i] = ao.shape[i] array.strides[i] = ao.strides[i] @@ -464,8 +417,8 @@ array.dataptr = direct_arrayitems(array.data) return array -def ll_transpose(ARRAY, BASE, a1): - a2 = ll_build_alias(ARRAY, BASE, a1) +def ll_transpose(ARRAY, a1): + a2 = ll_build_alias(ARRAY, a1) # XX do something to a2 return a2 Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Tue Sep 4 21:09:39 2007 @@ -182,10 +182,7 @@ a = t.buildannotator() s_array = a.build_types(f, []) assert type(s_array) == SomeArray - assert type(s_array.s_base) == SomeArray assert s_array.ndim == 1 - assert s_array.s_base.ndim == 2 - assert s_array.s_base.s_base == None from pypy.objspace.flow.model import checkgraph, flatten, Block, mkentrymap @@ -258,15 +255,6 @@ res = interpret(f, []) assert res == 1 - def test_specialize_base(self): - def f(): - a = numpy.empty((2,)) - b = a[:,] - return b.base is a - - res = interpret(f, []) - assert res - def test_specialize_array_attr_shape(self): def f(): a = numpy.empty((2,3)) From simonb at codespeak.net Tue Sep 4 21:15:34 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Tue, 4 Sep 2007 21:15:34 +0200 (CEST) Subject: [pypy-svn] r46310 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070904191534.8D5F58116@code0.codespeak.net> Author: simonb Date: Tue Sep 4 21:15:34 2007 New Revision: 46310 Modified: pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: fix array constructor Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Tue Sep 4 21:15:34 2007 @@ -352,8 +352,6 @@ class __extend__(pairtype(AbstractBaseListRepr, ArrayRepr)): def convert_from_to((r_lst, r_arr), v, llops): - #import py;py.test.skip() - assert 0 if r_lst.listitem is None: return NotImplemented if r_lst.item_repr != r_arr.item_repr: @@ -364,8 +362,6 @@ class __extend__(pairtype(AbstractRangeRepr, ArrayRepr)): def convert_from_to((r_rng, r_arr), v, llops): - #import py;py.test.skip() - assert 0 cARRAY = inputconst(lltype.Void, r_arr.lowleveltype.TO) return llops.gendirectcall(ll_build_from_list, cARRAY, v) Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Tue Sep 4 21:15:34 2007 @@ -22,16 +22,6 @@ test_c_compile = True test_llvm_compile = False -def fromlist1(lst, dtype=''): - if dtype: - a = numpy.empty((len(lst),), dtype=dtype) - else: - a = numpy.empty((len(lst),)) - idx = 0 - while idx < len(lst): - a[idx] = lst[idx] - idx += 1 - return a def access_array(item): my_array = numpy.array([item]) @@ -109,22 +99,20 @@ assert s.ndim == 2 def test_annotate_array_add(self): - py.test.skip() def f(): - a1 = fromlist1([1,2]) - a2 = fromlist1([6,9]) + a1 = numpy.array([1,2]) + a2 = numpy.array([6,9]) return a1 + a2 t = TranslationContext() a = t.buildannotator() s = a.build_types(f, []) - #assert s.typecode == 'i' + assert s.typecode == 'i' def test_annotate_array_add_coerce(self): - py.test.skip() def f(): - a1 = fromlist1([1,2]) - a2 = fromlist1([6.,9.]) + a1 = numpy.array([1,2]) + a2 = numpy.array([6.,9.]) return a1 + a2 t = TranslationContext() @@ -134,7 +122,7 @@ def test_annotate_array_dtype(self): def f(): - a1 = fromlist1([1,2], dtype='d') + a1 = numpy.array([1,2], dtype='d') return a1 t = TranslationContext() @@ -144,8 +132,8 @@ def test_annotate_array_array(self): def f(): - a1 = fromlist1([1,2], dtype='d') - a2 = fromlist1(a1) + a1 = numpy.array([1,2], dtype='d') + a2 = numpy.array(a1) return a2 t = TranslationContext() @@ -155,7 +143,7 @@ def test_annotate_array_attr(self): def f(): - a1 = fromlist1([1,2]) + a1 = numpy.array([1,2]) return a1.shape t = TranslationContext() @@ -165,7 +153,7 @@ def test_annotate_array_method(self): def f(): - a1 = fromlist1([1,2]) + a1 = numpy.array([1,2]) return a1.transpose() t = TranslationContext() @@ -206,7 +194,7 @@ # a[0] = 1 # a[1] = 20 # return a - a = fromlist1([1,20]) + a = numpy.array([1,20]) # b = numpy.array(a) # XX b = a return b @@ -225,7 +213,7 @@ def test_specialize_array_access(self): def access_with_variable(): - my_array = fromlist1(range(10), dtype='i') + my_array = numpy.array(range(10), dtype='i') my_array[2] = 2 sum = 0 for idx in range(10): @@ -237,10 +225,9 @@ assert res == 45 def test_specialize_array_add(self): - py.test.skip() def f(): - a1 = fromlist1([1.,2.]) - a2 = fromlist1([6,9]) + a1 = numpy.array([1.,2.]) + a2 = numpy.array([6,9]) return a1 + a2 res = interpret(f, []) @@ -249,7 +236,7 @@ def test_specialize_array_attr(self): def f(): - a = fromlist1([1,2]) + a = numpy.array([1,2]) return a.ndim res = interpret(f, []) @@ -315,7 +302,7 @@ py.test.skip('this test requires _always_inline_ magic hook') def f(): a = numpy.empty((3,), dtype='i') - b = fromlist1([5,55,555], dtype='i') + b = numpy.array([5,55,555], dtype='i') a[:] = b return a t = TranslationContext() @@ -344,7 +331,7 @@ def test_compile_array_access(self): def access_array(index): a = numpy.empty((3,), dtype='i') - b = fromlist1([5,55,555], dtype='i') + b = numpy.array([5,55,555], dtype='i') a[:] = b a[0] = 1 return a[index] From simonb at codespeak.net Wed Sep 5 02:48:05 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Wed, 5 Sep 2007 02:48:05 +0200 (CEST) Subject: [pypy-svn] r46313 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070905004805.75F1D8122@code0.codespeak.net> Author: simonb Date: Wed Sep 5 02:48:03 2007 New Revision: 46313 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: views are working well now Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Wed Sep 5 02:48:03 2007 @@ -69,6 +69,7 @@ raise AnnotatorError() return SomeArray(typecode, s_arr1.ndim) + # union ? sub = mul = div = add @@ -172,7 +173,7 @@ def specialize_call(self, hop, i_dtype=None): r_array = hop.r_result v_lst = hop.inputarg(r_array, 0) # coerce list arg to array arg - v_result = r_array.build_from_array(hop.llops, v_lst) + v_result = r_array.build_from_array(hop, v_lst) return v_result @@ -180,15 +181,17 @@ "Annotation and rtyping of calls to numpy.empty" _about_ = numpy.empty - def compute_result_annotation(self, s_tuple, s_dtype=None): - if isinstance(s_tuple, SomeTuple): + def compute_result_annotation(self, s_arg, s_dtype=None): + if isinstance(s_arg, SomeTuple): + s_tuple = s_arg for s_item in s_tuple.items: if not isinstance(s_item, SomeInteger): - raise AnnotatorError("shape must be tuple of integers") + raise AnnotatorError("shape arg not understood") ndim = len(s_tuple.items) + elif isinstance(s_arg, SomeInteger): + ndim = 1 else: - # XX also build from single int arg - raise AnnotatorError("could not build array shape from %s"%s_list) + raise AnnotatorError("shape arg not understood") typecode = 'd' if isinstance(s_dtype, SomeChar) and s_dtype.is_constant(): @@ -196,25 +199,25 @@ s_dtype = None return SomeArray(typecode, ndim) -# def specialize_call(self, hop): -# ldef = listdef.ListDef(None, SomeInteger()) -# r_lst = hop.rtyper.getrepr(SomeList(ldef)) -# # XX TyperError: don't know how to convert from -# # to -# # -# [v_lst] = hop.inputargs(r_lst) -# r_array = hop.r_result -# v_result = r_array.build_from_shape(hop.llops, r_lst, v_lst) -# return v_result - def specialize_call(self, hop, i_dtype=None): - r_tpl = hop.args_r[0] + r_arg = hop.args_r[0] # XX also call with single int arg - v_tpl = hop.inputarg(r_tpl, 0) + v_arg = hop.inputarg(r_arg, 0) r_array = hop.r_result - v_result = r_array.build_from_shape(hop.llops, r_tpl, v_tpl) + v_result = r_array.build_from_shape(hop, r_arg, v_arg) return v_result +class ZerosCallEntry(EmptyCallEntry): + "Annotation and rtyping of calls to numpy.zeros" + _about_ = numpy.zeros + + def specialize_call(self, hop, i_dtype=None): + r_arg = hop.args_r[0] + # XX also call with single int arg + v_arg = hop.inputarg(r_arg, 0) + r_array = hop.r_result + v_result = r_array.build_from_shape(hop, r_arg, v_arg, zero=True) + return v_result # Importing for side effect of registering types with extregistry Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Wed Sep 5 02:48:03 2007 @@ -18,7 +18,7 @@ from pypy.annotation import listdef from pypy.rpython.memory.lltypelayout import sizeof -def gen_build_from_shape(ndim): +def gen_build_from_shape(ndim, zero=False): unrolling_dims = unrolling_iterable(reversed(range(ndim))) def ll_build_from_shape(ARRAY, shape): array = ll_allocate(ARRAY, ndim) @@ -29,7 +29,7 @@ array.shape[i] = size array.strides[i] = itemsize itemsize *= size - array.data = malloc(ARRAY.data.TO, itemsize) + array.data = malloc(ARRAY.data.TO, itemsize, zero=zero) array.dataptr = direct_arrayitems(array.data) return array return ll_build_from_shape @@ -58,7 +58,7 @@ ("strides", INDEXARRAY), ("backstrides", INDEXARRAY), #("factors", INDEXARRAY), -# ("ao", ARRAY), # EECK!! this makes too many iter types !! + #("ao", ARRAY), # not needed.. ("dataptr", ARRAY.TO.dataptr), # pointer to current item #("contiguous", Bool), )) @@ -147,16 +147,26 @@ p0[0] = op(p1[0], p2[0]) def ll_array_unary_op(iter_new, iter_next, ITER, array0, array1): + assert array0.ndim == array1.ndim it0 = iter_new(ITER, array0) it1 = iter_new(ITER, array1) + assert it0.size == it1.size while it0.index < it0.size: - ll_unary_op(it0.dataptr, it1.dataptr) + it0.dataptr[0] = it1.dataptr[0] iter_next(it0) iter_next(it1) +def dim_of_ITER(ITER): + return ITER.TO.coordinates.length + def dim_of_ARRAY(ARRAY): return ARRAY.TO.shape.length +class ArrayIterRepr(Repr): + def __init__(self, rtyper, s_iter): + self.s_iter = s_iter + self.lowleveltype = self.ITER + class ArrayRepr(Repr): def __init__(self, rtyper, s_array): self.s_array = s_array @@ -180,19 +190,26 @@ self.lowleveltype = self.ARRAY self.ITER = ARRAY_ITER(self.ARRAY, self.INDEXARRAY) - def build_from_array(self, llops, v_array): + def build_from_array(self, hop, v_array): cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) - return llops.gendirectcall(ll_build_alias, cARRAY, v_array) + return hop.llops.gendirectcall(ll_build_alias, cARRAY, v_array) - def build_from_shape(self, llops, r_tuple, v_tuple): + def build_from_shape(self, hop, r_arg, v_arg, zero=False): cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) - cTUPLE = inputconst(lltype.Void, r_tuple.lowleveltype.TO) ndim = self.s_array.ndim - ll_build_from_shape = gen_build_from_shape(ndim) - c_ndim = inputconst(lltype.Signed, ndim) - assert ndim == len(r_tuple.items_r) - rval = llops.gendirectcall(ll_build_from_shape, cARRAY, v_tuple) - return rval + if isinstance(r_arg, TupleRepr): + r_tuple, v_tuple = r_arg, v_arg + cTUPLE = inputconst(lltype.Void, r_tuple.lowleveltype.TO) + ll_build_from_shape = gen_build_from_shape(ndim, zero) + c_ndim = inputconst(lltype.Signed, ndim) + assert ndim == len(r_tuple.items_r) + return hop.llops.gendirectcall(ll_build_from_shape, cARRAY, v_tuple) + else: + assert ndim == 1 + v_size = hop.inputarg(Signed, 0) + _malloc = lambda tp, size: malloc(tp, size, zero=zero) + cmalloc = inputconst(Void, _malloc) + return hop.llops.gendirectcall(ll_build_from_size, cARRAY, v_size, cmalloc) def rtype_method_transpose(self, hop): [v_self] = hop.inputargs(self) @@ -291,11 +308,11 @@ for src_i, r_item in unroll_r_tpl: if isinstance(r_item, IntegerRepr): r_int = r_item - dataptr = direct_ptradd(dataptr, getattr(tpl, 'item%d'%src_i)) + dataptr = direct_ptradd(dataptr, getattr(tpl, 'item%d'%src_i)*ao.strides[src_i]) else: r_slice = r_item array.shape[tgt_i] = ao.shape[src_i] - array.strides[tgt_i] = ao.strides[tgt_i] + array.strides[tgt_i] = ao.strides[src_i] tgt_i += 1 assert tgt_i == ndim array.dataptr = dataptr @@ -325,15 +342,18 @@ v_array, v_tuple, v_item = hop.inputargs(r_arr, r_tpl, r_item) ndim = get_view_ndim(r_tpl) if isinstance(r_item, ArrayRepr): + s_view = SomeArray(r_arr.s_array.typecode, ndim) + r_view = hop.rtyper.getrepr(s_view) + cARRAY = hop.inputconst(Void, r_view.ARRAY.TO) get_view = gen_get_view(r_tpl) - cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_tuple) - iter_new, iter_next = gen_iter_funcs(r_arr.ndim) + iter_new, iter_next = gen_iter_funcs(ndim) assert ndim == r_item.ndim + assert dim_of_ITER(r_item.ITER) == dim_of_ITER(r_view.ITER) cnew = hop.inputconst(Void, iter_new) cnext = hop.inputconst(Void, iter_next) cITER = hop.inputconst(Void, r_item.ITER.TO) - return hop.gendirectcall(ll_array_unary_op, cnew, cnext, cITER, v_array, v_view) + return hop.gendirectcall(ll_array_unary_op, cnew, cnext, cITER, v_view, v_item) else: # Set from scalar assert ndim == 0 @@ -344,10 +364,6 @@ class __extend__(pairtype(ArrayRepr, ArrayRepr)): def convert_from_to((r_arr0, r_arr1), v, llops): - print "%r -> %r"%( - (r_arr0, ), - (r_arr1, ), - ) assert 0 class __extend__(pairtype(AbstractBaseListRepr, ArrayRepr)): @@ -372,6 +388,14 @@ array.dataptr = nullptr(ARRAY.dataptr.TO) return array +def ll_build_from_size(ARRAY, size, _malloc): + array = ll_allocate(ARRAY, 1) + array.shape[0] = size + array.strides[0] = 1 + array.data = _malloc(ARRAY.data.TO, size) + array.dataptr = direct_arrayitems(array.data) + return array + def ll_build_from_list(ARRAY, lst): size = lst.ll_length() array = ll_allocate(ARRAY, 1) Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Wed Sep 5 02:48:03 2007 @@ -190,19 +190,22 @@ def test_specialize_array_create(self): def f(): -# a = numpy.empty((2,)) -# a[0] = 1 -# a[1] = 20 -# return a a = numpy.array([1,20]) -# b = numpy.array(a) # XX - b = a + b = numpy.array(a) return b res = interpret(f, []) assert res.data[0] == 1 assert res.data[1] == 20 + def test_specialize_array_empty1(self): + def f(n): + a = numpy.empty(n) + return a + + res = interpret(f, [3]) + assert res.ndim == 1 + def test_specialize_array_empty(self): def f(n, m): a = numpy.empty((n, m)) @@ -211,6 +214,16 @@ res = interpret(f, [3, 4]) assert res.ndim == 2 + def test_specialize_array_zeros(self): + def f(n, m): + a = numpy.zeros((n, m)) + return a + + res = interpret(f, [3, 4]) + for i in range(3*4): + assert res.dataptr[i] == 0 + assert res.ndim == 2 + def test_specialize_array_access(self): def access_with_variable(): my_array = numpy.array(range(10), dtype='i') @@ -266,14 +279,14 @@ def test_specialize_array_method(self): def f(): - a = numpy.empty((2,)) + a = numpy.array([1,2]) return a.transpose() res = interpret(f, []) -# assert res.data[0] == 1 -# assert res.data[1] == 2 + assert res.data[0] == 1 + assert res.data[1] == 2 - def test_specialize_indexing(self): + def test_specialize_view_0(self): def f(): a = numpy.empty((4,3), dtype='i') a[0,0] = 5 @@ -287,6 +300,7 @@ assert res.dataptr[6] == 555 assert res.shape.item0 == 4 assert res.strides.item0 == 3 + assert res.ndim == 1 def test_specialize_multi(self): def f(ii, jj): @@ -298,6 +312,19 @@ assert interpret(f, [0, 0]) == 0 assert interpret(f, [3, 4]) == 12 + def test_specialize_view(self): + def f(ii, jj): + a = numpy.zeros((4, 5)) + b = numpy.zeros((3, 4)) + a[0,1] = 5. + a[1,1] = 4. + a[2,1] = 3. + a[3,1] = 2. + b[2,:] = a[:,1] + return b[ii, jj] + + assert interpret(f, [2, 3]) == 2 + def test_malloc_remove(self): py.test.skip('this test requires _always_inline_ magic hook') def f(): @@ -326,7 +353,7 @@ py.test.skip("c compilation disabled") from pypy.translator.c.test.test_genc import compile - self.compile = lambda s, x, y : compile(x, y) + self.compile = lambda s, x, y : compile(x, y, backendopt=True) def test_compile_array_access(self): def access_array(index): @@ -352,4 +379,14 @@ fn = self.compile(f, [int, int]) assert fn(2,3) == 2*3 + def test_compile_view(self): + def f(ii, jj): + a = numpy.zeros((4, 5), dtype='i') + b = numpy.zeros((3, 4), dtype='i') + b[0,:] = a[:,0] + return b[ii, jj] + + fn = self.compile(f, [int, int]) + assert fn(2,3) == 0 + From arigo at codespeak.net Wed Sep 5 11:32:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 11:32:26 +0200 (CEST) Subject: [pypy-svn] r46314 - pypy/branch/merging Message-ID: <20070905093226.28C438123@code0.codespeak.net> Author: arigo Date: Wed Sep 5 11:32:25 2007 New Revision: 46314 Added: pypy/branch/merging/ - copied from r46313, pypy/dist/pypy/ Log: A temporary branch to hold the result of the merging of the rffi branch. From arigo at codespeak.net Wed Sep 5 11:35:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 11:35:41 +0200 (CEST) Subject: [pypy-svn] r46315 - pypy/branch/merging/module/thread Message-ID: <20070905093541.B1E6E8123@code0.codespeak.net> Author: arigo Date: Wed Sep 5 11:35:41 2007 New Revision: 46315 Added: pypy/branch/merging/module/thread/ - copied from r46314, pypy/branch/pypy-more-rtti-inprogress/module/thread/ Log: module/thread: the changes in the trunk were merged early into the branch. Overwrite the trunk version with the branch version. From mwh at codespeak.net Wed Sep 5 11:41:51 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 5 Sep 2007 11:41:51 +0200 (CEST) Subject: [pypy-svn] r46316 - pypy/extradoc/talk/pyconuk07 Message-ID: <20070905094151.B0A01812A@code0.codespeak.net> Author: mwh Date: Wed Sep 5 11:41:51 2007 New Revision: 46316 Modified: pypy/extradoc/talk/pyconuk07/pyconuk07.pdf Log: updates Modified: pypy/extradoc/talk/pyconuk07/pyconuk07.pdf ============================================================================== Binary files. No diff available. From arigo at codespeak.net Wed Sep 5 12:33:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:17 +0200 (CEST) Subject: [pypy-svn] r46317 - pypy/branch/merging/translator Message-ID: <20070905103317.342D78104@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:16 2007 New Revision: 46317 Added: pypy/branch/merging/translator/translator.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/translator/translator.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/translator/translator.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46028 | antocuni | 2007-08-27 15:51:13 +0200 (Mon, 27 Aug 2007) | 3 lines merge exceptiontransform from dist ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:19 +0200 (CEST) Subject: [pypy-svn] r46318 - pypy/branch/merging/rpython Message-ID: <20070905103319.0108A811D@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:19 2007 New Revision: 46318 Added: pypy/branch/merging/rpython/llinterp.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/rpython/llinterp.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/llinterp.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46087 | fijal | 2007-08-28 12:55:48 +0200 (Tue, 28 Aug 2007) | 4 lines Add malloc checking feature to llinterpreter, only works for malloc='raw', not sure if debugging info stored there makes sense, but well... we'll see. ------------------------------------------------------------------------ r46028 | antocuni | 2007-08-27 15:51:13 +0200 (Mon, 27 Aug 2007) | 3 lines merge exceptiontransform from dist ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:23 +0200 (CEST) Subject: [pypy-svn] r46319 - pypy/branch/merging/config Message-ID: <20070905103323.A5D0F8123@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:22 2007 New Revision: 46319 Added: pypy/branch/merging/config/translationoption.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/config/translationoption.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/config/translationoption.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46032 | antocuni | 2007-08-27 16:08:52 +0200 (Mon, 27 Aug 2007) | 3 lines merge translator/{cli,jvm,oosupport} and config from dist ------------------------------------------------------------------------ r46028 | antocuni | 2007-08-27 15:51:13 +0200 (Mon, 27 Aug 2007) | 3 lines merge exceptiontransform from dist ------------------------------------------------------------------------ r46021 | antocuni | 2007-08-27 14:55:29 +0200 (Mon, 27 Aug 2007) | 3 lines merge dist to branch (only gencli-related changes) ------------------------------------------------------------------------ r45933 | arigo | 2007-08-23 15:41:45 +0200 (Thu, 23 Aug 2007) | 2 lines So, --thread forces gc=boehm for now. ------------------------------------------------------------------------ r45928 | arigo | 2007-08-23 14:25:34 +0200 (Thu, 23 Aug 2007) | 3 lines Try to clarify the memory allocation situation around thread.start_new_thread(). ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:26 +0200 (CEST) Subject: [pypy-svn] r46320 - pypy/branch/merging/module/posix Message-ID: <20070905103326.B4AC38126@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:26 2007 New Revision: 46320 Added: pypy/branch/merging/module/posix/__init__.py.merge.tmp - copied unchanged from r46315, pypy/branch/merging/module/posix/__init__.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/module/posix/__init__.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46146 | arigo | 2007-08-29 13:13:22 +0200 (Wed, 29 Aug 2007) | 2 lines No os.getuid()/geteuid() on Windows. ------------------------------------------------------------------------ r45986 | exarkun | 2007-08-25 17:05:20 +0200 (Sat, 25 Aug 2007) | 1 line implement os.times for posix ------------------------------------------------------------------------ r45697 | arigo | 2007-08-16 09:15:31 +0200 (Thu, 16 Aug 2007) | 8 lines A more complete support for os.stat(). The RPython one now returns a Controlled object that supports both tuple-like indexing and the st_xxx attributes. The extra attributes are then exposed at app-level. Note that support for sub-second timestamps is there, but possibly a bit fragile. I disabled it because in CPython 2.4 by default we don't get sub-second timestamps out of os.stat() either. ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:28 +0200 (CEST) Subject: [pypy-svn] r46321 - pypy/branch/merging/module/posix Message-ID: <20070905103328.37DFE8126@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:28 2007 New Revision: 46321 Added: pypy/branch/merging/module/posix/interp_posix.py.merge.tmp - copied unchanged from r46315, pypy/branch/merging/module/posix/interp_posix.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/module/posix/interp_posix.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46019 | arigo | 2007-08-27 14:01:38 +0200 (Mon, 27 Aug 2007) | 3 lines In object spaces used during translation to OO backends, only expose the portable os.stat_result fields. ------------------------------------------------------------------------ r45986 | exarkun | 2007-08-25 17:05:20 +0200 (Sat, 25 Aug 2007) | 1 line implement os.times for posix ------------------------------------------------------------------------ r45828 | arigo | 2007-08-17 20:12:49 +0200 (Fri, 17 Aug 2007) | 2 lines don't capture the environment in the translated pypy ------------------------------------------------------------------------ r45710 | arigo | 2007-08-16 13:09:54 +0200 (Thu, 16 Aug 2007) | 2 lines Some more large file support: os.ftruncate(), os.lseek(). ------------------------------------------------------------------------ r45697 | arigo | 2007-08-16 09:15:31 +0200 (Thu, 16 Aug 2007) | 8 lines A more complete support for os.stat(). The RPython one now returns a Controlled object that supports both tuple-like indexing and the st_xxx attributes. The extra attributes are then exposed at app-level. Note that support for sub-second timestamps is there, but possibly a bit fragile. I disabled it because in CPython 2.4 by default we don't get sub-second timestamps out of os.stat() either. ------------------------------------------------------------------------ r45680 | arigo | 2007-08-15 15:35:51 +0200 (Wed, 15 Aug 2007) | 2 lines Missed the KeyError on del os.environ['unknown_name'] ------------------------------------------------------------------------ r45679 | arigo | 2007-08-15 15:03:44 +0200 (Wed, 15 Aug 2007) | 3 lines - support for 'os.environ.get(name)'. - kill rlib/ros.py. Yay! ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:31 +0200 (CEST) Subject: [pypy-svn] r46322 - pypy/branch/merging/objspace/flow Message-ID: <20070905103331.1207D812E@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:31 2007 New Revision: 46322 Added: pypy/branch/merging/objspace/flow/objspace.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/objspace/flow/objspace.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/objspace/flow/objspace.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46258 | arigo | 2007-09-03 11:13:18 +0200 (Mon, 03 Sep 2007) | 2 lines time.time() -> win ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:35 +0200 (CEST) Subject: [pypy-svn] r46323 - pypy/branch/merging/translator/test Message-ID: <20070905103335.2C77A811D@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:34 2007 New Revision: 46323 Added: pypy/branch/merging/translator/test/test_exceptiontransform.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/translator/test/test_exceptiontransform.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/translator/test/test_exceptiontransform.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46029 | antocuni | 2007-08-27 16:00:56 +0200 (Mon, 27 Aug 2007) | 3 lines restore this "poor test, lost in the dark" (cit.) ------------------------------------------------------------------------ r46023 | fijal | 2007-08-27 15:30:16 +0200 (Mon, 27 Aug 2007) | 2 lines Fix test ------------------------------------------------------------------------ r45997 | pedronis | 2007-08-26 18:13:26 +0200 (Sun, 26 Aug 2007) | 6 lines llexternals typically should not raise. have a canraise flag on them for the benefit of the raise analyzer and exception transform. this changeset will be fun to merge because on the trunk the exception transform has been generalized to work with ootype and has moved. be careful. ------------------------------------------------------------------------ r45992 | pedronis | 2007-08-26 13:47:59 +0200 (Sun, 26 Aug 2007) | 3 lines show the problem with the wrong assumption of exception transform vs. llexternals ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:37 +0200 (CEST) Subject: [pypy-svn] r46324 - pypy/branch/merging/translator/cli Message-ID: <20070905103337.A91BC8125@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:37 2007 New Revision: 46324 Added: pypy/branch/merging/translator/cli/metavm.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/translator/cli/metavm.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/translator/cli/metavm.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46045 | antocuni | 2007-08-27 17:55:41 +0200 (Mon, 27 Aug 2007) | 3 lines implement cast_primitive and ullong_lshift for gencli ------------------------------------------------------------------------ r46032 | antocuni | 2007-08-27 16:08:52 +0200 (Mon, 27 Aug 2007) | 3 lines merge translator/{cli,jvm,oosupport} and config from dist ------------------------------------------------------------------------ r46030 | antocuni | 2007-08-27 16:02:57 +0200 (Mon, 27 Aug 2007) | 3 lines revert commit 46021: I started merging from the wrong revision ------------------------------------------------------------------------ r46021 | antocuni | 2007-08-27 14:55:29 +0200 (Mon, 27 Aug 2007) | 3 lines merge dist to branch (only gencli-related changes) ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:40 +0200 (CEST) Subject: [pypy-svn] r46325 - pypy/branch/merging/rpython/lltypesystem Message-ID: <20070905103340.824D28131@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:40 2007 New Revision: 46325 Added: pypy/branch/merging/rpython/lltypesystem/lltype.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/rpython/lltypesystem/lltype.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/lltype.py revisions 45507 to 46315: ------------------------------------------------------------------------ r45879 | arigo | 2007-08-20 16:57:24 +0200 (Mon, 20 Aug 2007) | 17 lines Fix for an issue where multiple _func objects have the same graph. They were created by multiple calls to rtyper.getcallable(), once without particular arguments, and once from annlowlevel.py with extra **fnobjattrs. In details: * getcallable() now always reads extra fnobj attributes from the Python function object's '_fnobjattrs_' attribute, if any, so that it always return an equal _func object. * this allows us to revert the recently-introduced **fnobjattrs arguments in annlowlevel.py. * make _func objects really immutable, to avoid surprizes with hash(). * shuffle things around in ll2ctypes to cope with this. translator/c/test/test_newgc is now passing. ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:43 +0200 (CEST) Subject: [pypy-svn] r46326 - pypy/branch/merging/rpython/test Message-ID: <20070905103343.EB2258135@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:43 2007 New Revision: 46326 Added: pypy/branch/merging/rpython/test/test_llinterp.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/rpython/test/test_llinterp.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_llinterp.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46087 | fijal | 2007-08-28 12:55:48 +0200 (Tue, 28 Aug 2007) | 4 lines Add malloc checking feature to llinterpreter, only works for malloc='raw', not sure if debugging info stored there makes sense, but well... we'll see. ------------------------------------------------------------------------ r46028 | antocuni | 2007-08-27 15:51:13 +0200 (Mon, 27 Aug 2007) | 3 lines merge exceptiontransform from dist ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:46 +0200 (CEST) Subject: [pypy-svn] r46327 - pypy/branch/merging/rpython/test Message-ID: <20070905103346.B54AA8104@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:46 2007 New Revision: 46327 Added: pypy/branch/merging/rpython/test/test_rfloat.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/rpython/test/test_rfloat.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_rfloat.py revisions 45507 to 46315: ------------------------------------------------------------------------ r45964 | arigo | 2007-08-24 18:59:54 +0200 (Fri, 24 Aug 2007) | 2 lines Test and fix: float division doesn't raise ZeroDivisionError in RPython. ------------------------------------------------------------------------ r45960 | fijal | 2007-08-24 18:42:05 +0200 (Fri, 24 Aug 2007) | 2 lines Revert float % float, as arigo suggested ------------------------------------------------------------------------ r45945 | fijal | 2007-08-24 11:48:40 +0200 (Fri, 24 Aug 2007) | 2 lines Implement float % float ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:49 +0200 (CEST) Subject: [pypy-svn] r46328 - pypy/branch/merging/rpython/module Message-ID: <20070905103349.62E868136@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:48 2007 New Revision: 46328 Added: pypy/branch/merging/rpython/module/ll_os.py.merge.tmp - copied unchanged from r46315, pypy/branch/merging/rpython/module/ll_os.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46290 | arigo | 2007-09-03 18:28:37 +0200 (Mon, 03 Sep 2007) | 3 lines Sprinkling underscores in front of C library function names on Windows, for ctypes to find the functions. ------------------------------------------------------------------------ r46286 | arigo | 2007-09-03 17:55:29 +0200 (Mon, 03 Sep 2007) | 2 lines Fix os.listdir() on Linux ------------------------------------------------------------------------ r46281 | arigo | 2007-09-03 17:07:15 +0200 (Mon, 03 Sep 2007) | 2 lines Fix os.utime() for Linux. ------------------------------------------------------------------------ r46280 | arigo | 2007-09-03 17:02:35 +0200 (Mon, 03 Sep 2007) | 4 lines os.utime(): * Windows support * sub-second resolution support (not for Windows) ------------------------------------------------------------------------ r46274 | arigo | 2007-09-03 14:27:46 +0200 (Mon, 03 Sep 2007) | 2 lines More progress, mostly test fixing. ------------------------------------------------------------------------ r46268 | arigo | 2007-09-03 13:40:26 +0200 (Mon, 03 Sep 2007) | 2 lines Windozify listdir() and pipe(). ------------------------------------------------------------------------ r46266 | arigo | 2007-09-03 12:38:56 +0200 (Mon, 03 Sep 2007) | 3 lines Make the RPython interface to os.mkdir() always take a mode argument, which we just ignore on Windows (CPython's os.mkdir() behaves like that too). ------------------------------------------------------------------------ r46247 | santagada | 2007-09-02 23:53:12 +0200 (Sun, 02 Sep 2007) | 2 lines in the wait manpage for os x and freebsd this is the right place for the signal header. ------------------------------------------------------------------------ r46244 | arigo | 2007-09-02 19:56:09 +0200 (Sun, 02 Sep 2007) | 2 lines test_ll_os.py passes on Windows. ------------------------------------------------------------------------ r46243 | arigo | 2007-09-02 19:50:02 +0200 (Sun, 02 Sep 2007) | 2 lines Start again with Windows porting. ------------------------------------------------------------------------ r46193 | fijal | 2007-08-30 14:39:49 +0200 (Thu, 30 Aug 2007) | 2 lines Fix translation ------------------------------------------------------------------------ r46188 | fijal | 2007-08-30 12:36:38 +0200 (Thu, 30 Aug 2007) | 2 lines Use more cool helpers ------------------------------------------------------------------------ r46185 | fijal | 2007-08-30 12:15:02 +0200 (Thu, 30 Aug 2007) | 2 lines Use cool helpers ------------------------------------------------------------------------ r46154 | arigo | 2007-08-29 13:45:11 +0200 (Wed, 29 Aug 2007) | 2 lines In-progress Windowsification. I hope I didn't break it on Linux. ------------------------------------------------------------------------ r46146 | arigo | 2007-08-29 13:13:22 +0200 (Wed, 29 Aug 2007) | 2 lines No os.getuid()/geteuid() on Windows. ------------------------------------------------------------------------ r45986 | exarkun | 2007-08-25 17:05:20 +0200 (Sat, 25 Aug 2007) | 1 line implement os.times for posix ------------------------------------------------------------------------ r45981 | fijal | 2007-08-25 14:32:54 +0200 (Sat, 25 Aug 2007) | 2 lines Fix name ------------------------------------------------------------------------ r45980 | pedronis | 2007-08-25 13:58:18 +0200 (Sat, 25 Aug 2007) | 3 lines support an idiom such that in the best case the original function needs to specified only in the @registering decorator ------------------------------------------------------------------------ r45979 | pedronis | 2007-08-25 13:19:16 +0200 (Sat, 25 Aug 2007) | 8 lines registering_if convenience decorator for the pattern: if hasattr(ns, name): @registering_if(ns.name) def ... ------------------------------------------------------------------------ r45976 | pedronis | 2007-08-25 12:51:46 +0200 (Sat, 25 Aug 2007) | 2 lines shorter names: lltypeimpl -> llimpl ------------------------------------------------------------------------ r45812 | arigo | 2007-08-17 15:05:56 +0200 (Fri, 17 Aug 2007) | 12 lines Change the level at which --sandbox'ing occurs. Now, by default the functions registered by _register_external() are replaced with a sandboxing version, so that the low-level external functions should no longer be seen at all. The _register_external()'ed functions operate at a slightly higher level - they have regular RPython objects as arguments and result. This is a more natural level that can be processed directly by pypy.rlib.rmarshal with no special-casing needed. This allows us to get rid of the strange marshal_input/unmarshal_output hacks in ll_os.py. As an extra bonus, the protocol used between the sandboxed process and its parent is now based on the regular CPython marshal format. ------------------------------------------------------------------------ r45735 | fijal | 2007-08-16 17:14:03 +0200 (Thu, 16 Aug 2007) | 3 lines Rename CXxx to CXxxPtr and make Cxx return xx (CStruct now returns a struct, not a pointer to this and so on) ------------------------------------------------------------------------ r45734 | arigo | 2007-08-16 16:56:52 +0200 (Thu, 16 Aug 2007) | 2 lines os._exit(). That was the last one, kill a couple of unnecessary files, yay! ------------------------------------------------------------------------ r45732 | arigo | 2007-08-16 16:46:47 +0200 (Thu, 16 Aug 2007) | 2 lines os.spawnv()... with a new test which I cannot run on Linux, so well, untested ------------------------------------------------------------------------ r45731 | arigo | 2007-08-16 16:34:59 +0200 (Thu, 16 Aug 2007) | 2 lines os.fork()... ------------------------------------------------------------------------ r45730 | arigo | 2007-08-16 16:32:52 +0200 (Thu, 16 Aug 2007) | 3 lines os.link()... os.symlink()... ------------------------------------------------------------------------ r45729 | arigo | 2007-08-16 16:27:20 +0200 (Thu, 16 Aug 2007) | 2 lines os.kill()... ------------------------------------------------------------------------ r45728 | arigo | 2007-08-16 16:21:08 +0200 (Thu, 16 Aug 2007) | 2 lines os.umask()... ------------------------------------------------------------------------ r45727 | arigo | 2007-08-16 16:19:34 +0200 (Thu, 16 Aug 2007) | 2 lines os.rename()... ------------------------------------------------------------------------ r45726 | arigo | 2007-08-16 16:17:57 +0200 (Thu, 16 Aug 2007) | 2 lines os.chmod()... ------------------------------------------------------------------------ r45725 | arigo | 2007-08-16 16:15:01 +0200 (Thu, 16 Aug 2007) | 4 lines os.chdir()... os.mkdir()... os.rmdir()... ------------------------------------------------------------------------ r45724 | arigo | 2007-08-16 16:06:42 +0200 (Thu, 16 Aug 2007) | 2 lines os.unlink()... ------------------------------------------------------------------------ r45723 | arigo | 2007-08-16 16:04:43 +0200 (Thu, 16 Aug 2007) | 2 lines os.system()... ------------------------------------------------------------------------ r45722 | arigo | 2007-08-16 15:59:07 +0200 (Thu, 16 Aug 2007) | 2 lines os.strerror()... ------------------------------------------------------------------------ r45721 | arigo | 2007-08-16 15:54:46 +0200 (Thu, 16 Aug 2007) | 2 lines os.isatty()... ------------------------------------------------------------------------ r45717 | arigo | 2007-08-16 14:08:13 +0200 (Thu, 16 Aug 2007) | 2 lines os.ftruncate(). Test large files when translated to C too. ------------------------------------------------------------------------ r45700 | arigo | 2007-08-16 11:40:30 +0200 (Thu, 16 Aug 2007) | 2 lines os.lseek -> rffi ------------------------------------------------------------------------ r45697 | arigo | 2007-08-16 09:15:31 +0200 (Thu, 16 Aug 2007) | 8 lines A more complete support for os.stat(). The RPython one now returns a Controlled object that supports both tuple-like indexing and the st_xxx attributes. The extra attributes are then exposed at app-level. Note that support for sub-second timestamps is there, but possibly a bit fragile. I disabled it because in CPython 2.4 by default we don't get sub-second timestamps out of os.stat() either. ------------------------------------------------------------------------ r45679 | arigo | 2007-08-15 15:03:44 +0200 (Wed, 15 Aug 2007) | 3 lines - support for 'os.environ.get(name)'. - kill rlib/ros.py. Yay! ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:51 +0200 (CEST) Subject: [pypy-svn] r46329 - pypy/branch/merging/module/posix/test Message-ID: <20070905103351.F3D078135@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:51 2007 New Revision: 46329 Added: pypy/branch/merging/module/posix/test/test_posix2.py.merge.tmp - copied unchanged from r46315, pypy/branch/merging/module/posix/test/test_posix2.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46302 | arigo | 2007-09-04 13:04:55 +0200 (Tue, 04 Sep 2007) | 2 lines Skip on Windows. ------------------------------------------------------------------------ r46300 | arigo | 2007-09-04 12:47:37 +0200 (Tue, 04 Sep 2007) | 2 lines Skip tests for Windows. ------------------------------------------------------------------------ r46197 | fijal | 2007-08-30 16:05:49 +0200 (Thu, 30 Aug 2007) | 2 lines Damn. Make this check anything. Thanks xoraxax ------------------------------------------------------------------------ r46000 | pedronis | 2007-08-26 23:19:59 +0200 (Sun, 26 Aug 2007) | 3 lines hfs+ (default os x fs) has no sparse files ------------------------------------------------------------------------ r45986 | exarkun | 2007-08-25 17:05:20 +0200 (Sat, 25 Aug 2007) | 1 line implement os.times for posix ------------------------------------------------------------------------ r45710 | arigo | 2007-08-16 13:09:54 +0200 (Thu, 16 Aug 2007) | 2 lines Some more large file support: os.ftruncate(), os.lseek(). ------------------------------------------------------------------------ r45697 | arigo | 2007-08-16 09:15:31 +0200 (Thu, 16 Aug 2007) | 8 lines A more complete support for os.stat(). The RPython one now returns a Controlled object that supports both tuple-like indexing and the st_xxx attributes. The extra attributes are then exposed at app-level. Note that support for sub-second timestamps is there, but possibly a bit fragile. I disabled it because in CPython 2.4 by default we don't get sub-second timestamps out of os.stat() either. ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:54 +0200 (CEST) Subject: [pypy-svn] r46330 - pypy/branch/merging/translator/jvm/test Message-ID: <20070905103354.69E158147@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:53 2007 New Revision: 46330 Added: pypy/branch/merging/translator/jvm/test/test_cast.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/translator/jvm/test/test_cast.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/translator/jvm/test/test_cast.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46045 | antocuni | 2007-08-27 17:55:41 +0200 (Mon, 27 Aug 2007) | 3 lines implement cast_primitive and ullong_lshift for gencli ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:33:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:33:57 +0200 (CEST) Subject: [pypy-svn] r46331 - pypy/branch/merging/translator/cli/src Message-ID: <20070905103357.0A0398126@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:33:57 2007 New Revision: 46331 Added: pypy/branch/merging/translator/cli/src/pypylib.cs.merge.tmp - copied, changed from r46315, pypy/branch/merging/translator/cli/src/pypylib.cs Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/translator/cli/src/pypylib.cs revisions 45507 to 46315: ------------------------------------------------------------------------ r46063 | antocuni | 2007-08-27 21:17:46 +0200 (Mon, 27 Aug 2007) | 3 lines fix some signature mismatch. pypy-cli works again on the branch/ ------------------------------------------------------------------------ r46062 | antocuni | 2007-08-27 21:03:10 +0200 (Mon, 27 Aug 2007) | 3 lines make os.environ work again with gencli ------------------------------------------------------------------------ r46032 | antocuni | 2007-08-27 16:08:52 +0200 (Mon, 27 Aug 2007) | 3 lines merge translator/{cli,jvm,oosupport} and config from dist ------------------------------------------------------------------------ r46030 | antocuni | 2007-08-27 16:02:57 +0200 (Mon, 27 Aug 2007) | 3 lines revert commit 46021: I started merging from the wrong revision ------------------------------------------------------------------------ r46021 | antocuni | 2007-08-27 14:55:29 +0200 (Mon, 27 Aug 2007) | 3 lines merge dist to branch (only gencli-related changes) ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:34:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:34:00 +0200 (CEST) Subject: [pypy-svn] r46332 - pypy/branch/merging/rpython/numpy/test Message-ID: <20070905103400.84E418150@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:34:00 2007 New Revision: 46332 Added: pypy/branch/merging/rpython/numpy/test/test_array.py.merge.tmp - copied, changed from r46315, pypy/branch/merging/rpython/numpy/test/test_array.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/numpy/test/test_array.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46095 | fijal | 2007-08-28 13:31:21 +0200 (Tue, 28 Aug 2007) | 2 lines *really* skip ------------------------------------------------------------------------ r46093 | fijal | 2007-08-28 13:25:26 +0200 (Tue, 28 Aug 2007) | 2 lines Skip this test if numpy is not there ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:34:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:34:04 +0200 (CEST) Subject: [pypy-svn] r46333 - pypy/branch/merging/rpython/module/test Message-ID: <20070905103404.1CC968106@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:34:03 2007 New Revision: 46333 Added: pypy/branch/merging/rpython/module/test/test_ll_os.py.merge.tmp - copied unchanged from r46315, pypy/branch/merging/rpython/module/test/test_ll_os.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_ll_os.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46244 | arigo | 2007-09-02 19:56:09 +0200 (Sun, 02 Sep 2007) | 2 lines test_ll_os.py passes on Windows. ------------------------------------------------------------------------ r45986 | exarkun | 2007-08-25 17:05:20 +0200 (Sat, 25 Aug 2007) | 1 line implement os.times for posix ------------------------------------------------------------------------ r45733 | arigo | 2007-08-16 16:56:09 +0200 (Thu, 16 Aug 2007) | 2 lines Forgot to fix these tests. ------------------------------------------------------------------------ r45679 | arigo | 2007-08-15 15:03:44 +0200 (Wed, 15 Aug 2007) | 3 lines - support for 'os.environ.get(name)'. - kill rlib/ros.py. Yay! ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:34:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:34:05 +0200 (CEST) Subject: [pypy-svn] r46334 - pypy/branch/merging/rpython/module/test Message-ID: <20070905103405.F11618126@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:34:05 2007 New Revision: 46334 Added: pypy/branch/merging/rpython/module/test/test_posix.py.merge.tmp - copied unchanged from r46315, pypy/branch/merging/rpython/module/test/test_posix.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-more-rtti-inprogress/rpython/module/test/test_posix.py revisions 45507 to 46315: ------------------------------------------------------------------------ r46291 | arigo | 2007-09-03 18:32:30 +0200 (Mon, 03 Sep 2007) | 2 lines Fix tests. ------------------------------------------------------------------------ r45986 | exarkun | 2007-08-25 17:05:20 +0200 (Sat, 25 Aug 2007) | 1 line implement os.times for posix ------------------------------------------------------------------------ r45656 | arigo | 2007-08-14 18:10:44 +0200 (Tue, 14 Aug 2007) | 6 lines For backup purposes, in-progress: convert more of the os module to rtti and try to get rid of the rllib.ros module by making the native interfaces RPythonic. This looks quite good in my opinion - seems that we've finally learned a reasonable way to do external functions. ------------------------------------------------------------------------ From arigo at codespeak.net Wed Sep 5 12:35:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 12:35:20 +0200 (CEST) Subject: [pypy-svn] r46335 - in pypy/branch/merging: annotation annotation/test config interpreter jit/codegen module/_curses module/posix module/sys module/time objspace/flow rlib rlib/parsing rlib/rcairo/test rlib/test rpython rpython/lltypesystem rpython/lltypesystem/module rpython/lltypesystem/test rpython/memory/gctransform/test rpython/memory/test rpython/module rpython/numpy/test rpython/ootypesystem/module rpython/ootypesystem/test rpython/test rpython/tool rpython/tool/test translator translator/backendopt translator/c translator/c/src translator/c/test translator/cli translator/cli/src translator/cli/test translator/js/modules translator/jvm/test translator/llvm/test translator/oosupport/test_template translator/sandbox translator/test translator/tool Message-ID: <20070905103520.220448128@code0.codespeak.net> Author: arigo Date: Wed Sep 5 12:35:18 2007 New Revision: 46335 Added: pypy/branch/merging/annotation/binaryop.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/annotation/binaryop.py pypy/branch/merging/annotation/signature.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/annotation/signature.py pypy/branch/merging/annotation/test/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/annotation/test/ pypy/branch/merging/config/translationoption.py - copied unchanged from r46334, pypy/branch/merging/config/translationoption.py.merge.tmp pypy/branch/merging/interpreter/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/interpreter/ pypy/branch/merging/jit/codegen/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/jit/codegen/ pypy/branch/merging/module/_curses/fficurses.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/module/_curses/fficurses.py pypy/branch/merging/module/posix/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/module/posix/ pypy/branch/merging/module/sys/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/module/sys/ pypy/branch/merging/module/time/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/module/time/ pypy/branch/merging/objspace/flow/objspace.py - copied unchanged from r46334, pypy/branch/merging/objspace/flow/objspace.py.merge.tmp pypy/branch/merging/rlib/_rsocket_rffi.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rlib/_rsocket_rffi.py pypy/branch/merging/rlib/parsing/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rlib/parsing/ pypy/branch/merging/rlib/rbigint.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rlib/rbigint.py pypy/branch/merging/rlib/rcairo/test/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rlib/rcairo/test/ pypy/branch/merging/rlib/rmarshal.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rlib/rmarshal.py pypy/branch/merging/rlib/test/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rlib/test/ pypy/branch/merging/rpython/annlowlevel.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/annlowlevel.py pypy/branch/merging/rpython/controllerentry.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/controllerentry.py pypy/branch/merging/rpython/extfunc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/extfunc.py pypy/branch/merging/rpython/extfuncregistry.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/extfuncregistry.py pypy/branch/merging/rpython/extfunctable.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/extfunctable.py pypy/branch/merging/rpython/llinterp.py - copied unchanged from r46334, pypy/branch/merging/rpython/llinterp.py.merge.tmp pypy/branch/merging/rpython/lltypesystem/ll2ctypes.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/ll2ctypes.py pypy/branch/merging/rpython/lltypesystem/lloperation.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/lloperation.py pypy/branch/merging/rpython/lltypesystem/lltype.py - copied unchanged from r46334, pypy/branch/merging/rpython/lltypesystem/lltype.py.merge.tmp pypy/branch/merging/rpython/lltypesystem/module/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/module/ pypy/branch/merging/rpython/lltypesystem/rffi.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rffi.py pypy/branch/merging/rpython/lltypesystem/rstr.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/rstr.py pypy/branch/merging/rpython/lltypesystem/test/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/lltypesystem/test/ pypy/branch/merging/rpython/memory/gctransform/test/test_refcounting.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/memory/gctransform/test/test_refcounting.py pypy/branch/merging/rpython/memory/test/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/memory/test/ pypy/branch/merging/rpython/module/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/module/ pypy/branch/merging/rpython/numpy/test/test_array.py - copied unchanged from r46334, pypy/branch/merging/rpython/numpy/test/test_array.py.merge.tmp pypy/branch/merging/rpython/ootypesystem/module/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/ootypesystem/module/ pypy/branch/merging/rpython/ootypesystem/test/test_bltann.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/ootypesystem/test/test_bltann.py pypy/branch/merging/rpython/rbool.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/rbool.py pypy/branch/merging/rpython/rbuiltin.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/rbuiltin.py pypy/branch/merging/rpython/rcontrollerentry.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/rcontrollerentry.py pypy/branch/merging/rpython/rfloat.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/rfloat.py pypy/branch/merging/rpython/rint.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/rint.py pypy/branch/merging/rpython/test/test_extfunc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_extfunc.py pypy/branch/merging/rpython/test/test_extfuncregister.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_extfuncregister.py pypy/branch/merging/rpython/test/test_llann.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_llann.py pypy/branch/merging/rpython/test/test_llinterp.py - copied unchanged from r46334, pypy/branch/merging/rpython/test/test_llinterp.py.merge.tmp pypy/branch/merging/rpython/test/test_nongc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_nongc.py pypy/branch/merging/rpython/test/test_rbuiltin.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_rbuiltin.py pypy/branch/merging/rpython/test/test_rfloat.py - copied unchanged from r46334, pypy/branch/merging/rpython/test/test_rfloat.py.merge.tmp pypy/branch/merging/rpython/test/test_rptr.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/test/test_rptr.py pypy/branch/merging/rpython/tool/genrffi.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/tool/genrffi.py pypy/branch/merging/rpython/tool/mkrffi.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/tool/mkrffi.py pypy/branch/merging/rpython/tool/rffi_platform.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/tool/rffi_platform.py pypy/branch/merging/rpython/tool/rfficache.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/tool/rfficache.py pypy/branch/merging/rpython/tool/test/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/tool/test/ pypy/branch/merging/rpython/typesystem.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/rpython/typesystem.py pypy/branch/merging/translator/backendopt/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/backendopt/ pypy/branch/merging/translator/c/database.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/database.py pypy/branch/merging/translator/c/extfunc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/extfunc.py pypy/branch/merging/translator/c/funcgen.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/funcgen.py pypy/branch/merging/translator/c/genc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/genc.py pypy/branch/merging/translator/c/node.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/node.py pypy/branch/merging/translator/c/src/float.h - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/src/float.h pypy/branch/merging/translator/c/src/ll_math.h - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/src/ll_math.h pypy/branch/merging/translator/c/src/ll_os.h - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/src/ll_os.h pypy/branch/merging/translator/c/src/stack.h - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/src/stack.h pypy/branch/merging/translator/c/src/support.h - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/src/support.h pypy/branch/merging/translator/c/src/thread.c - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/src/thread.c pypy/branch/merging/translator/c/src/thread_pthread.h - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/src/thread_pthread.h pypy/branch/merging/translator/c/test/test_extfunc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py pypy/branch/merging/translator/c/test/test_genc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_genc.py pypy/branch/merging/translator/c/test/test_lltyped.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_lltyped.py pypy/branch/merging/translator/cli/database.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/cli/database.py pypy/branch/merging/translator/cli/metavm.py - copied unchanged from r46334, pypy/branch/merging/translator/cli/metavm.py.merge.tmp pypy/branch/merging/translator/cli/opcodes.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/cli/opcodes.py pypy/branch/merging/translator/cli/src/ll_os.cs - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/cli/src/ll_os.cs pypy/branch/merging/translator/cli/src/pypylib.cs - copied unchanged from r46334, pypy/branch/merging/translator/cli/src/pypylib.cs.merge.tmp pypy/branch/merging/translator/cli/test/test_builtin.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/cli/test/test_builtin.py pypy/branch/merging/translator/js/modules/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/js/modules/ pypy/branch/merging/translator/jvm/test/test_cast.py - copied unchanged from r46334, pypy/branch/merging/translator/jvm/test/test_cast.py.merge.tmp pypy/branch/merging/translator/llvm/test/test_extfunc.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/llvm/test/test_extfunc.py pypy/branch/merging/translator/oosupport/test_template/cast.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/oosupport/test_template/cast.py pypy/branch/merging/translator/oosupport/test_template/operations.py - copied unchanged from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/oosupport/test_template/operations.py pypy/branch/merging/translator/sandbox/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/sandbox/ pypy/branch/merging/translator/test/test_exceptiontransform.py - copied unchanged from r46334, pypy/branch/merging/translator/test/test_exceptiontransform.py.merge.tmp pypy/branch/merging/translator/tool/ - copied from r46334, pypy/branch/pypy-more-rtti-inprogress/translator/tool/ pypy/branch/merging/translator/translator.py - copied unchanged from r46334, pypy/branch/merging/translator/translator.py.merge.tmp Removed: pypy/branch/merging/config/translationoption.py.merge.tmp pypy/branch/merging/objspace/flow/objspace.py.merge.tmp pypy/branch/merging/rpython/llinterp.py.merge.tmp pypy/branch/merging/rpython/lltypesystem/lltype.py.merge.tmp pypy/branch/merging/rpython/numpy/test/test_array.py.merge.tmp pypy/branch/merging/rpython/test/test_llinterp.py.merge.tmp pypy/branch/merging/rpython/test/test_rexternalobj.py pypy/branch/merging/rpython/test/test_rfloat.py.merge.tmp pypy/branch/merging/translator/c/src/ll_osdefs.h pypy/branch/merging/translator/cli/metavm.py.merge.tmp pypy/branch/merging/translator/cli/src/pypylib.cs.merge.tmp pypy/branch/merging/translator/jvm/test/test_cast.py.merge.tmp pypy/branch/merging/translator/test/test_exceptiontransform.py.merge.tmp pypy/branch/merging/translator/translator.py.merge.tmp Log: (arigo, antocuni around) Finish merging the rffi branch into the "merging" branch. From arigo at codespeak.net Wed Sep 5 13:18:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 13:18:43 +0200 (CEST) Subject: [pypy-svn] r46336 - pypy/branch/merging/rpython/module/test Message-ID: <20070905111843.D033D811E@code0.codespeak.net> Author: arigo Date: Wed Sep 5 13:18:42 2007 New Revision: 46336 Modified: pypy/branch/merging/rpython/module/test/test_ll_os.py pypy/branch/merging/rpython/module/test/test_posix.py Log: Repeat r45631: move the test so that it is llinterpreted. Modified: pypy/branch/merging/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/branch/merging/rpython/module/test/test_ll_os.py (original) +++ pypy/branch/merging/rpython/module/test/test_ll_os.py Wed Sep 5 13:18:42 2007 @@ -48,18 +48,6 @@ assert file(filename).read().strip() == '2' os.unlink(filename) -def test_os_wstar(): - from pypy.rpython.module.ll_os import RegisterOs - for name in RegisterOs.w_star: - if not hasattr(os, name): - continue - def fun(s): - return getattr(os, name)(s) - - fn = compile(fun, [int]) - for value in [0, 1, 127, 128, 255]: - assert fn(value) == fun(value) - class ExpectTestOs: def setup_class(cls): if not hasattr(os, 'ttyname'): Modified: pypy/branch/merging/rpython/module/test/test_posix.py ============================================================================== --- pypy/branch/merging/rpython/module/test/test_posix.py (original) +++ pypy/branch/merging/rpython/module/test/test_posix.py Wed Sep 5 13:18:42 2007 @@ -118,6 +118,18 @@ return os.getuid() assert self.interpret(f, []) == f() + def test_os_wstar(self): + from pypy.rpython.module.ll_os import RegisterOs + for name in RegisterOs.w_star: + if not hasattr(os, name): + continue + def fun(s): + return getattr(os, name)(s) + + for value in [0, 1, 127, 128, 255]: + res = self.interpret(fun, [value]) + assert res == fun(value) + class TestLLtype(BaseTestPosix, LLRtypeMixin): if False and hasattr(os, 'uname'): def test_os_uname(self): From arigo at codespeak.net Wed Sep 5 13:26:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 13:26:42 +0200 (CEST) Subject: [pypy-svn] r46337 - in pypy/branch/merging: rlib rpython/lltypesystem Message-ID: <20070905112642.3B48180B6@code0.codespeak.net> Author: arigo Date: Wed Sep 5 13:26:41 2007 New Revision: 46337 Removed: pypy/branch/merging/rlib/ros.py pypy/branch/merging/rpython/lltypesystem/rfficache.py Log: Repeat the deletion of these two files that occurred on the pypy-more-rtti-inprogress branch. From arigo at codespeak.net Wed Sep 5 13:29:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 13:29:42 +0200 (CEST) Subject: [pypy-svn] r46338 - in pypy: branch/merging branch/pypy-more-rtti-inprogress dist/pypy Message-ID: <20070905112942.112708113@code0.codespeak.net> Author: arigo Date: Wed Sep 5 13:29:41 2007 New Revision: 46338 Added: pypy/dist/pypy/ - copied from r46337, pypy/branch/merging/ Removed: pypy/branch/merging/ pypy/branch/pypy-more-rtti-inprogress/ Log: Finish the merge of the pypy-more-rtti-inprogress branch. Delete the branches. From arigo at codespeak.net Wed Sep 5 13:41:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 13:41:47 +0200 (CEST) Subject: [pypy-svn] r46339 - in pypy/dist/pypy: rpython/module rpython/module/test translator/c/test Message-ID: <20070905114147.422448124@code0.codespeak.net> Author: arigo Date: Wed Sep 5 13:41:46 2007 New Revision: 46339 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_posix.py pypy/dist/pypy/translator/c/test/test_extfunc.py Log: Reimplement os.uname(). Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Wed Sep 5 13:41:46 2007 @@ -302,46 +302,40 @@ return extdef([], int, export_name="ll_os.ll_os_setsid", llimpl=setsid_llimpl) - if False: - @registering_if(os, 'uname') - def register_os_uname(self): - lgt = platform.intdefined('_UTSNAME_LENGTH', - includes=['sys/utsname.h']) - UTCHARP = lltype.FixedSizeArray(lltype.Char, lgt) - fields = [('sysname', UTCHARP), - ('nodename', UTCHARP), - ('release', UTCHARP), - ('version', UTCHARP), - ('machine', UTCHARP), - ('domainname', UTCHARP)] - UTSNAMEP = rffi.CStructPtr('utsname', *fields) - - os_uname = self.llexternal('uname', [UTSNAMEP], rffi.INT, - includes=['sys/utsname.h']) - - def utcharp2str(cp): - l = [] - i = 0 - while cp[i] != '\x00' and i < lgt: - l.append(cp[i]) - i += 1 - return "".join(l) - - def uname_llimpl(): - l_utsbuf = lltype.malloc(UTSNAMEP.TO, flavor='raw') - result = os_uname(l_utsbuf) - if result == -1: - raise OSError(rffi.get_errno(), "os_uname failed") - fields = [l_utsbuf.c_sysname, l_utsbuf.c_nodename, - l_utsbuf.c_release, l_utsbuf.c_version, - l_utsbuf.c_machine] - l = [utcharp2str(i) for i in fields] - retval = (l[0], l[1], l[2], l[3], l[4]) - lltype.free(l_utsbuf, flavor='raw') - return retval + @registering_if(os, 'uname') + def register_os_uname(self): + CHARARRAY = lltype.FixedSizeArray(lltype.Char, 1) + class CConfig: + _includes_ = ['sys/utsname.h'] + UTSNAME = platform.Struct('struct utsname', [ + ('sysname', CHARARRAY), + ('nodename', CHARARRAY), + ('release', CHARARRAY), + ('version', CHARARRAY), + ('machine', CHARARRAY)]) + config = platform.configure(CConfig) + UTSNAMEP = lltype.Ptr(config['UTSNAME']) + + os_uname = self.llexternal('uname', [UTSNAMEP], rffi.INT, + includes=CConfig._includes_) + + def uname_llimpl(): + l_utsbuf = lltype.malloc(UTSNAMEP.TO, flavor='raw') + result = os_uname(l_utsbuf) + if result == -1: + raise OSError(rffi.get_errno(), "os_uname failed") + retval = ( + rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_sysname)), + rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_nodename)), + rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_release)), + rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_version)), + rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_machine)), + ) + lltype.free(l_utsbuf, flavor='raw') + return retval - return extdef([], (str, str, str, str, str), - "ll_os.ll_uname", llimpl=uname_llimpl) + return extdef([], (str, str, str, str, str), + "ll_os.ll_uname", llimpl=uname_llimpl) @registering_if(os, 'getuid') def register_os_getuid(self): Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Wed Sep 5 13:41:46 2007 @@ -131,13 +131,7 @@ assert res == fun(value) class TestLLtype(BaseTestPosix, LLRtypeMixin): - if False and hasattr(os, 'uname'): - def test_os_uname(self): - for num in range(5): - def fun(): - return os.uname()[num] - res = self.interpret(fun, []) - assert self.ll_to_string(res) == os.uname()[num] + pass class TestOOtype(BaseTestPosix, OORtypeMixin): pass Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/c/test/test_extfunc.py Wed Sep 5 13:41:46 2007 @@ -918,3 +918,14 @@ assert os.stat(path).st_atime > t0 func(0) assert int(os.stat(path).st_atime) == int(t0) + +if hasattr(os, 'uname'): + def test_os_uname(): + def does_stuff(num): + tup = os.uname() + lst = [tup[0], tup[1], tup[2], tup[3], tup[4]] + return lst[num] + func = compile(does_stuff, [int]) + for i in range(5): + res = func(i) + assert res == os.uname()[i] From xoraxax at codespeak.net Wed Sep 5 13:49:24 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 5 Sep 2007 13:49:24 +0200 (CEST) Subject: [pypy-svn] r46340 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20070905114924.7BD598123@code0.codespeak.net> Author: xoraxax Date: Wed Sep 5 13:49:22 2007 New Revision: 46340 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Log: Do not bail out on missing ctypes (ignore the importerror), now pypy doesnt depend on ctypes for easy targets. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Wed Sep 5 13:49:22 2007 @@ -1,6 +1,11 @@ import sys -import ctypes -import ctypes.util + +try: + import ctypes + import ctypes.util +except ImportError: + ctypes = None + import os from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.extfunc import ExtRegistryEntry @@ -417,16 +422,17 @@ # __________ the standard C library __________ -if sys.platform == 'win32': - # trying to guess the correct libc... only a few tests fail if there - # is a mismatch between the one used by python2x.dll and the one - # loaded here - if sys.version_info < (2, 4): - standard_c_lib = ctypes.cdll.LoadLibrary('msvcrt.dll') +if ctypes: + if sys.platform == 'win32': + # trying to guess the correct libc... only a few tests fail if there + # is a mismatch between the one used by python2x.dll and the one + # loaded here + if sys.version_info < (2, 4): + standard_c_lib = ctypes.cdll.LoadLibrary('msvcrt.dll') + else: + standard_c_lib = ctypes.cdll.LoadLibrary('msvcr71.dll') else: - standard_c_lib = ctypes.cdll.LoadLibrary('msvcr71.dll') -else: - standard_c_lib = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c')) + standard_c_lib = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c')) # ____________________________________________ From arigo at codespeak.net Wed Sep 5 14:17:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 14:17:23 +0200 (CEST) Subject: [pypy-svn] r46341 - in pypy/dist/pypy: rlib rlib/test rpython/module Message-ID: <20070905121723.6659780E0@code0.codespeak.net> Author: arigo Date: Wed Sep 5 14:17:21 2007 New Revision: 46341 Modified: pypy/dist/pypy/rlib/rmarshal.py pypy/dist/pypy/rlib/test/test_rmarshal.py pypy/dist/pypy/rpython/module/ll_os_stat.py Log: Fix rlib.rmarshal. Modified: pypy/dist/pypy/rlib/rmarshal.py ============================================================================== --- pypy/dist/pypy/rlib/rmarshal.py (original) +++ pypy/dist/pypy/rlib/rmarshal.py Wed Sep 5 14:17:21 2007 @@ -8,7 +8,6 @@ from pypy.annotation.pairtype import pair, pairtype from pypy.rlib.rarithmetic import formatd, r_longlong, intmask from pypy.rlib.unroll import unrolling_iterable -from pypy.rpython import controllerentry class CannotMarshal(Exception): pass @@ -303,10 +302,28 @@ class __extend__(pairtype(MTag, annmodel.SomeObject)): def install_marshaller((tag, s_obj)): - raise CannotMarshal(s_obj) + if not hasattr(s_obj, '_get_rmarshall_support_'): + raise CannotMarshal(s_obj) + # special support for custom annotation like SomeStatResult: + # the annotation tells us how to turn an object into something + # else that can be marshalled + def dump_with_custom_reduce(buf, x): + reduced_obj = fn_reduce(x) + reduceddumper(buf, reduced_obj) + s_reduced_obj, fn_reduce, fn_recreate = s_obj._get_rmarshall_support_() + reduceddumper = get_marshaller(s_reduced_obj) + add_dumper(s_obj, dump_with_custom_reduce) def install_unmarshaller((tag, s_obj)): - raise CannotUnmarshall(s_obj) + if not hasattr(s_obj, '_get_rmarshall_support_'): + raise CannotUnmarshall(s_obj) + # special support for custom annotation like SomeStatResult + def load_with_custom_recreate(loader): + reduced_obj = reducedloader(loader) + return fn_recreate(reduced_obj) + s_reduced_obj, fn_reduce, fn_recreate = s_obj._get_rmarshall_support_() + reducedloader = get_loader(s_reduced_obj) + add_loader(s_obj, load_with_custom_recreate) class __extend__(pairtype(MTag, annmodel.SomeList)): @@ -380,23 +397,24 @@ add_loader(s_tuple, load_tuple) -class __extend__(pairtype(MTag, controllerentry.SomeControlledInstance)): - # marshal a ControlledInstance by marshalling the underlying object - - def install_marshaller((tag, s_obj)): - def dump_controlled_instance(buf, x): - real_obj = controllerentry.controlled_instance_unbox(controller, x) - realdumper(buf, real_obj) - - controller = s_obj.controller - realdumper = get_marshaller(s_obj.s_real_obj) - add_dumper(s_obj, dump_controlled_instance) - - def install_unmarshaller((tag, s_obj)): - def load_controlled_instance(loader): - real_obj = realloader(loader) - return controllerentry.controlled_instance_box(controller, - real_obj) - controller = s_obj.controller - realloader = get_loader(s_obj.s_real_obj) - add_loader(s_obj, load_controlled_instance) +## -- not used any more right now -- +##class __extend__(pairtype(MTag, controllerentry.SomeControlledInstance)): +## # marshal a ControlledInstance by marshalling the underlying object + +## def install_marshaller((tag, s_obj)): +## def dump_controlled_instance(buf, x): +## real_obj = controllerentry.controlled_instance_unbox(controller, x) +## realdumper(buf, real_obj) + +## controller = s_obj.controller +## realdumper = get_marshaller(s_obj.s_real_obj) +## add_dumper(s_obj, dump_controlled_instance) + +## def install_unmarshaller((tag, s_obj)): +## def load_controlled_instance(loader): +## real_obj = realloader(loader) +## return controllerentry.controlled_instance_box(controller, +## real_obj) +## controller = s_obj.controller +## realloader = get_loader(s_obj.s_real_obj) +## add_loader(s_obj, load_controlled_instance) Modified: pypy/dist/pypy/rlib/test/test_rmarshal.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmarshal.py (original) +++ pypy/dist/pypy/rlib/test/test_rmarshal.py Wed Sep 5 14:17:21 2007 @@ -116,21 +116,10 @@ res = ''.join(res.chars) assert res == '5=hello;7=world;' -def test_tuple_stat_result(): - from pypy.rpython.module.ll_os_stat import STAT_FIELDS, s_tuple_StatResult - marshaller = get_marshaller(s_tuple_StatResult) - unmarshaller = get_unmarshaller(s_tuple_StatResult) - - sttuple = (0,) * len(STAT_FIELDS) - buf = [] - marshaller(buf, sttuple) - res = unmarshaller(''.join(buf)) - assert res == sttuple - def test_stat_result(): import os from pypy.translator.c.test.test_genc import compile - from pypy.rpython.module.ll_os_stat import STAT_FIELDS, s_StatResult + from pypy.rpython.module.ll_os_stat import s_StatResult marshal_stat_result = get_marshaller(s_StatResult) unmarshal_stat_result = get_unmarshaller(s_StatResult) def f(path): @@ -152,4 +141,4 @@ assert sttuple[3] == st[3] assert sttuple[4] == st[4] assert sttuple[5] == st[5] - assert len(sttuple) == len(STAT_FIELDS) + assert len(sttuple) == 10 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Wed Sep 5 14:17:21 2007 @@ -90,6 +90,19 @@ TYPE = STAT_FIELD_TYPES[attrname] return annmodel.lltype_to_annotation(TYPE) + def _get_rmarshall_support_(self): # for rlib.rmarshal + # reduce and recreate stat_result objects from 10-tuples + # (we ignore the extra values here for simplicity and portability) + def stat_result_reduce(st): + return (st[0], st[1], st[2], st[3], st[4], + st[5], st[6], st[7], st[8], st[9]) + def stat_result_recreate(tup): + return make_stat_result(tup + extra_zeroes) + s_reduced = annmodel.SomeTuple([annmodel.lltype_to_annotation(TYPE) + for name, TYPE in PORTABLE_STAT_FIELDS]) + extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS)) + return s_reduced, stat_result_reduce, stat_result_recreate + class __extend__(pairtype(SomeStatResult, annmodel.SomeInteger)): def getitem((s_sta, s_int)): assert s_int.is_constant(), "os.stat()[index]: index must be constant" From antocuni at codespeak.net Wed Sep 5 14:20:56 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 Sep 2007 14:20:56 +0200 (CEST) Subject: [pypy-svn] r46342 - pypy/dist/pypy/translator/jvm Message-ID: <20070905122056.6E35B80E5@code0.codespeak.net> Author: antocuni Date: Wed Sep 5 14:20:56 2007 New Revision: 46342 Modified: pypy/dist/pypy/translator/jvm/opcodes.py Log: ullong_rhift for genjvm Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Wed Sep 5 14:20:56 2007 @@ -213,6 +213,7 @@ 'ullong_floordiv': jvmgen.LDIV, # valid? 'ullong_mod': jvmgen.PYPYULONGMOD, 'ullong_mod_zer': _check_zer(jvmgen.PYPYULONGMOD), + 'ullong_rshift': [PushAllArgs, jvmgen.L2I, jvmgen.LUSHR, StoreResult], 'ullong_lt': 'ulong_less_than', 'ullong_le': 'ulong_less_equals', 'ullong_eq': 'ulong_equals', From arigo at codespeak.net Wed Sep 5 14:23:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 14:23:07 +0200 (CEST) Subject: [pypy-svn] r46343 - in pypy/dist/pypy/translator/sandbox: . test Message-ID: <20070905122307.6B51080EA@code0.codespeak.net> Author: arigo Date: Wed Sep 5 14:23:06 2007 New Revision: 46343 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Log: Fix the sandlib. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Wed Sep 5 14:23:06 2007 @@ -6,7 +6,7 @@ import py import marshal, sys, os, posixpath, errno, stat -from pypy.rpython.module.ll_os_stat import STAT_FIELDS, s_tuple_StatResult +from pypy.rpython.module.ll_os_stat import s_StatResult from pypy.tool.ansi_print import AnsiLog from pypy.rlib.rarithmetic import r_longlong from py.compat import subprocess @@ -273,16 +273,10 @@ log.vpath('%r => %r' % (vpath, node)) return node - def build_stat_result(self, st): - result = tuple(st) - result += (0,) * (len(STAT_FIELDS) - len(result)) - return result - def do_ll_os__ll_os_stat(self, vpathname): node = self.get_node(vpathname) - st = node.stat() - return self.build_stat_result(st) - do_ll_os__ll_os_stat.resulttype = s_tuple_StatResult + return node.stat() + do_ll_os__ll_os_stat.resulttype = s_StatResult do_ll_os__ll_os_lstat = do_ll_os__ll_os_stat Modified: pypy/dist/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Wed Sep 5 14:23:06 2007 @@ -77,7 +77,7 @@ assert tail == "" def test_sandbox_4(): - from pypy.rpython.module.ll_os_stat import STAT_FIELDS, s_tuple_StatResult + from pypy.rpython.module.ll_os_stat import s_StatResult from pypy.rlib.rarithmetic import r_longlong r0x12380000007 = r_longlong(0x12380000007) @@ -89,10 +89,9 @@ t = Translation(entry_point, backend='c', standalone=True, sandbox=True) exe = t.compile() g, f = os.popen2(exe, "t", 0) - sttuple = (55, 0, 0, 0, 0, 0, 0x12380000007, 0, 0, 0) - sttuple += (0,) * (len(STAT_FIELDS)-len(sttuple)) - expect(f, g, "ll_os.ll_os_stat", ("somewhere",), sttuple, - resulttype = s_tuple_StatResult) + st = os.stat_result((55, 0, 0, 0, 0, 0, 0x12380000007, 0, 0, 0)) + expect(f, g, "ll_os.ll_os_stat", ("somewhere",), st, + resulttype = s_StatResult) expect(f, g, "ll_os.ll_os_ftruncate", (55, 0x12380000007), None) g.close() tail = f.read() From arigo at codespeak.net Wed Sep 5 14:50:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 14:50:37 +0200 (CEST) Subject: [pypy-svn] r46344 - in pypy/dist/pypy: rpython/lltypesystem translator/c translator/c/test Message-ID: <20070905125037.656EE80DD@code0.codespeak.net> Author: arigo Date: Wed Sep 5 14:50:35 2007 New Revision: 46344 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/translator/c/primitive.py pypy/dist/pypy/translator/c/test/test_lltyped.py Log: A test for prebuilt integers of various types in genc. The fix is to make it pass even when ctypes is not installed, and removing a dependency on the rctypes package. Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Sep 5 14:50:35 2007 @@ -107,6 +107,7 @@ def setup(): """ creates necessary c-level types """ + result = [] for name in TYPES: c_name = name if name.startswith('unsigned'): @@ -118,8 +119,10 @@ tp = platform.inttype(name.upper(), c_name, signed) globals()['r_' + name] = platform.numbertype_to_rclass[tp] globals()[name.upper()] = tp + result.append(tp) + return result -setup() +NUMBER_TYPES = setup() platform.numbertype_to_rclass[lltype.Signed] = int # avoid "r_long" for common cases # ^^^ this creates at least the following names: # -------------------------------------------------------------------- Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Wed Sep 5 14:50:35 2007 @@ -2,6 +2,7 @@ from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rpython.lltypesystem.lltype import * +from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, WeakGcAddress, fakeweakaddress, \ @@ -175,20 +176,14 @@ PrimitiveType[ll_type] = '%s @'% c_name PrimitiveErrorValue[ll_type] = '((%s) -1)'% c_name -try: - import ctypes -except ImportError: - pass -else: - from pypy.rpython.rctypes import rcarithmetic as rcarith - for ll_type, c_name in [(rcarith.CByte, 'signed char'), - (rcarith.CUByte, 'unsigned char'), - (rcarith.CShort, 'short'), - (rcarith.CUShort, 'unsigned short'), - (rcarith.CInt, 'int'), - (rcarith.CUInt, 'unsigned int'), - (rcarith.CLong, 'long'), - (rcarith.CULong, 'unsigned long'), - (rcarith.CLonglong, 'long long'), - (rcarith.CULonglong, 'unsigned long long')]: - define_c_primitive(ll_type, c_name) +for ll_type, c_name in [(rffi.SIGNEDCHAR, 'signed char'), + (rffi.UCHAR, 'unsigned char'), + (rffi.SHORT, 'short'), + (rffi.USHORT, 'unsigned short'), + (rffi.INT, 'int'), + (rffi.UINT, 'unsigned int'), + (rffi.LONG, 'long'), + (rffi.ULONG, 'unsigned long'), + (rffi.LONGLONG, 'long long'), + (rffi.ULONGLONG, 'unsigned long long')]: + define_c_primitive(ll_type, c_name) Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Wed Sep 5 14:50:35 2007 @@ -318,3 +318,25 @@ fn = self.getcompiled(f, [int]) res = fn(14) assert res == -1789569707 + + def test_prebuilt_integers(self): + from pypy.rlib.unroll import unrolling_iterable + from pypy.rpython.lltypesystem import rffi + class Prebuilt: + pass + p = Prebuilt() + NUMBER_TYPES = rffi.NUMBER_TYPES + names = unrolling_iterable([TYPE.__name__ for TYPE in NUMBER_TYPES]) + for name, TYPE in zip(names, NUMBER_TYPES): + value = cast_primitive(TYPE, 1) + setattr(p, name, value) + + def f(x): + total = x + for name in names: + total += rffi.cast(Signed, getattr(p, name)) + return total + + fn = self.getcompiled(f, [int]) + res = fn(100) + assert res == 100 + len(list(names)) From antocuni at codespeak.net Wed Sep 5 15:20:24 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 Sep 2007 15:20:24 +0200 (CEST) Subject: [pypy-svn] r46345 - in pypy/dist/pypy/translator: cli/test jvm/test oosupport/test_template Message-ID: <20070905132024.11FAF8115@code0.codespeak.net> Author: antocuni Date: Wed Sep 5 15:20:24 2007 New Revision: 46345 Added: pypy/dist/pypy/translator/oosupport/test_template/runtest.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/test/test_runtest.py pypy/dist/pypy/translator/jvm/test/test_runtest.py Log: port test_runtest to oosupport, and remove duplicated code Modified: pypy/dist/pypy/translator/cli/test/test_runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/test_runtest.py Wed Sep 5 15:20:24 2007 @@ -1,74 +1,11 @@ import py +from pypy.translator.oosupport.test_template.runtest import BaseTestRunTest from pypy.translator.cli.test.runtest import CliTest -from pypy.translator.cli.test.runtest import FLOAT_PRECISION -from pypy.annotation.listdef import s_list_of_strings -def ident(x): - return x - -class TestRunTest(CliTest): - - def test_patch_os(self): - import os - from pypy.translator.cli.support import patch, unpatch, NT_OS - original_O_CREAT = os.O_CREAT - olddefs = patch() - assert os.O_CREAT == NT_OS['O_CREAT'] - unpatch(*olddefs) - assert os.O_CREAT == original_O_CREAT - - def test_int(self): - assert self.interpret(ident, [42]) == 42 - - def test_bool(self): - assert self.interpret(ident, [True]) == True - assert self.interpret(ident, [False]) == False - - def test_float(self): - x = 10/3.0 - res = self.interpret(ident, [x]) - assert self.float_eq(x, res) - - def test_char(self): - assert self.interpret(ident, ['a']) == 'a' - - def test_list(self): - def fn(): - return [1, 2, 3] - assert self.interpret(fn, []) == [1, 2, 3] - - def test_tuple(self): - def fn(): - return 1, 2 - assert self.interpret(fn, []) == (1, 2) - - def test_string(self): - def fn(): - return 'foo' - res = self.interpret(fn, []) - assert self.ll_to_string(res) == 'foo' - - def test_exception(self): - def fn(): - raise ValueError - self.interpret_raises(ValueError, fn, []) - - def test_exception_subclass(self): - def fn(): - raise IndexError - self.interpret_raises(LookupError, fn, []) - - def test_object_or_none(self): - def fn(flag): - if flag: - return "hello"; - else: - return None - assert self.interpret(fn, [False]) is None +class TestRunTest(BaseTestRunTest, CliTest): def test_auto_raise_exc(self): def fn(): raise ValueError f = self._compile(fn, [], auto_raise_exc=True) py.test.raises(ValueError, f) - Modified: pypy/dist/pypy/translator/jvm/test/test_runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_runtest.py Wed Sep 5 15:20:24 2007 @@ -1,71 +1,5 @@ -import sys +from pypy.translator.oosupport.test_template.runtest import BaseTestRunTest from pypy.translator.jvm.test.runtest import JvmTest -from pypy.translator.jvm.test.runtest import FLOAT_PRECISION -from pypy.annotation.listdef import s_list_of_strings -from pypy.rlib.rarithmetic import r_ulonglong -def ident(x): - return x - -class TestRunTest(JvmTest): - - def test_patch_os(self): - import os - from pypy.translator.cli.support import patch, unpatch, NT_OS - original_O_CREAT = os.O_CREAT - olddefs = patch() - assert os.O_CREAT == NT_OS['O_CREAT'] - unpatch(*olddefs) - assert os.O_CREAT == original_O_CREAT - - def test_int(self): - assert self.interpret(ident, [42]) == 42 - - def test_bool(self): - assert self.interpret(ident, [True]) == True - assert self.interpret(ident, [False]) == False - - def test_float(self): - x = 10/3.0 - res = self.interpret(ident, [x]) - assert self.float_eq(x, res) - - def test_char(self): - assert self.interpret(ident, ['a']) == 'a' - - def test_list(self): - def fn(): - return [1, 2, 3] - assert self.interpret(fn, []) == [1, 2, 3] - - def test_tuple(self): - def fn(): - return 1, 2 - assert self.interpret(fn, []) == (1, 2) - - def test_string(self): - def fn(): - return 'foo' - res = self.interpret(fn, []) - assert self.ll_to_string(res) == 'foo' - - def test_exception(self): - def fn(): - raise ValueError - self.interpret_raises(ValueError, fn, []) - - def test_exception_subclass(self): - def fn(): - raise IndexError - self.interpret_raises(LookupError, fn, []) - - def test_object_or_none(self): - def fn(flag): - if flag: - return "hello"; - else: - return None - assert self.interpret(fn, [False]) is None - - def test_ullong(self): - assert self.interpret(ident, [r_ulonglong(sys.maxint+1)]) == sys.maxint+1 +class TestRunTest(BaseTestRunTest, JvmTest): + pass Added: pypy/dist/pypy/translator/oosupport/test_template/runtest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/test_template/runtest.py Wed Sep 5 15:20:24 2007 @@ -0,0 +1,71 @@ +import sys +import py +from pypy.annotation.listdef import s_list_of_strings +from pypy.rlib.rarithmetic import r_ulonglong + +def ident(x): + return x + +class BaseTestRunTest: + + def test_patch_os(self): + import os + from pypy.translator.cli.support import patch, unpatch, NT_OS + original_O_CREAT = os.O_CREAT + olddefs = patch() + assert os.O_CREAT == NT_OS['O_CREAT'] + unpatch(*olddefs) + assert os.O_CREAT == original_O_CREAT + + def test_int(self): + assert self.interpret(ident, [42]) == 42 + + def test_bool(self): + assert self.interpret(ident, [True]) == True + assert self.interpret(ident, [False]) == False + + def test_float(self): + x = 10/3.0 + res = self.interpret(ident, [x]) + assert self.float_eq(x, res) + + def test_char(self): + assert self.interpret(ident, ['a']) == 'a' + + def test_list(self): + def fn(): + return [1, 2, 3] + assert self.interpret(fn, []) == [1, 2, 3] + + def test_tuple(self): + def fn(): + return 1, 2 + assert self.interpret(fn, []) == (1, 2) + + def test_string(self): + def fn(): + return 'foo' + res = self.interpret(fn, []) + assert self.ll_to_string(res) == 'foo' + + def test_ullong(self): + assert self.interpret(ident, [r_ulonglong(sys.maxint+1)]) == sys.maxint+1 + + def test_exception(self): + def fn(): + raise ValueError + self.interpret_raises(ValueError, fn, []) + + def test_exception_subclass(self): + def fn(): + raise IndexError + self.interpret_raises(LookupError, fn, []) + + def test_object_or_none(self): + def fn(flag): + if flag: + return "hello"; + else: + return None + assert self.interpret(fn, [False]) is None + From arigo at codespeak.net Wed Sep 5 16:07:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 16:07:02 +0200 (CEST) Subject: [pypy-svn] r46347 - in pypy/dist/pypy/module/posix: . test Message-ID: <20070905140702.132F9811F@code0.codespeak.net> Author: arigo Date: Wed Sep 5 16:07:01 2007 New Revision: 46347 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/app_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: This is a replacement for the revision 45516 and 45517: a pure app-level implementation of os.tmpfile() based on the 'tempfile' module. The idea is that it nicely avoids the hacks that were needed in 45516 and 45517. Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Wed Sep 5 16:07:01 2007 @@ -17,6 +17,7 @@ 'error' : 'app_posix.error', 'stat_result': 'app_posix.stat_result', 'fdopen' : 'app_posix.fdopen', + 'tmpfile' : 'app_posix.tmpfile', } interpleveldefs = { Modified: pypy/dist/pypy/module/posix/app_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/app_posix.py (original) +++ pypy/dist/pypy/module/posix/app_posix.py Wed Sep 5 16:07:01 2007 @@ -6,8 +6,10 @@ import sys if 'posix' in sys.builtin_module_names: import posix + osname = 'posix' elif 'nt' in sys.builtin_module_names: import nt as posix + osname = 'nt' else: raise ImportError("XXX") @@ -56,6 +58,19 @@ return file.fdopen(fd, mode, buffering) +def tmpfile(): + """Create a temporary file. + + The data in the file is freed when you + close the file, or automatically by the OS when the program ends.""" + import tempfile + f = tempfile.TemporaryFile() + if osname == 'nt': + f = f.file # on NT, with the 2.4 stdlib of CPython, + # we get a _TemporaryFileWrapper for no good reason + return f + + # __________ only if we have os.fork() __________ class popenfile(file): Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Wed Sep 5 16:07:01 2007 @@ -328,6 +328,15 @@ res = os.system(cmd) assert res == 0 + def test_tmpfile(self): + os = self.os + f = os.tmpfile() + f.write("xxx") + f.flush() + f.seek(0, 0) + assert isinstance(f, file) + assert f.read() == 'xxx' + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible def setup_class(cls): From arigo at codespeak.net Wed Sep 5 18:07:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 18:07:49 +0200 (CEST) Subject: [pypy-svn] r46348 - pypy/dist/pypy/rpython/module Message-ID: <20070905160749.139988132@code0.codespeak.net> Author: arigo Date: Wed Sep 5 18:07:48 2007 New Revision: 46348 Modified: pypy/dist/pypy/rpython/module/ll_os.py Log: clock_t is unsigned on OS/X - missing a cast. Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Wed Sep 5 18:07:48 2007 @@ -276,7 +276,7 @@ l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw') try: result = os_times(l_tmsbuf) - if result == -1: + if result == rffi.cast(self.CLOCK_T, -1): raise OSError(rffi.get_errno(), "times failed") return ( l_tmsbuf.c_tms_utime / CLOCK_TICKS_PER_SECOND, From arigo at codespeak.net Wed Sep 5 18:14:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 18:14:02 +0200 (CEST) Subject: [pypy-svn] r46349 - pypy/dist/pypy/objspace/flow Message-ID: <20070905161402.6DC068132@code0.codespeak.net> Author: arigo Date: Wed Sep 5 18:14:01 2007 New Revision: 46349 Modified: pypy/dist/pypy/objspace/flow/objspace.py Log: Oups, there is a test for this function. Keep the old signature valid. Modified: pypy/dist/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/objspace.py (original) +++ pypy/dist/pypy/objspace/flow/objspace.py Wed Sep 5 18:14:01 2007 @@ -563,7 +563,7 @@ OverflowError) # for the float case del _add_exceptions, _add_except_ovf -def extract_cell_content(c, varname, func): +def extract_cell_content(c, varname='?', func='?'): """Get the value contained in a CPython 'cell', as read through the func_closure of a function object.""" # yuk! this is all I could come up with that works in Python 2.2 too From arigo at codespeak.net Wed Sep 5 18:17:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 18:17:49 +0200 (CEST) Subject: [pypy-svn] r46350 - pypy/dist/pypy/translator/llvm/test Message-ID: <20070905161749.70B1A8132@code0.codespeak.net> Author: arigo Date: Wed Sep 5 18:17:49 2007 New Revision: 46350 Modified: pypy/dist/pypy/translator/llvm/test/runtest.py Log: Skip all llvm tests for now... Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Wed Sep 5 18:17:49 2007 @@ -1,4 +1,5 @@ import py +py.test.skip("llvm is no longer actively maintained") from pypy.tool import isolate from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM From arigo at codespeak.net Wed Sep 5 18:27:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 18:27:57 +0200 (CEST) Subject: [pypy-svn] r46352 - pypy/dist/pypy/translator/test Message-ID: <20070905162757.EC164812D@code0.codespeak.net> Author: arigo Date: Wed Sep 5 18:27:57 2007 New Revision: 46352 Modified: pypy/dist/pypy/translator/test/test_simplify.py Log: These two tests were too precise. Modified: pypy/dist/pypy/translator/test/test_simplify.py ============================================================================== --- pypy/dist/pypy/translator/test/test_simplify.py (original) +++ pypy/dist/pypy/translator/test/test_simplify.py Wed Sep 5 18:27:57 2007 @@ -29,7 +29,7 @@ def f(x): os.close(x) graph, _ = translate(f, [int]) - assert len(graph.startblock.operations) == 2 + assert len(graph.startblock.operations) > 0 def test_remove_recursive_call(): def rec(a): @@ -140,13 +140,24 @@ print op graph = get_graph(op.args[0], t) assert graph is not None + # an external function in RPython turns currently into + # a call to a wrapper function which itself contains the + # real call to a graph-less external ll function, so + # we check recursively graph, t = translate(external_function, [], False) - for block in graph.iterblocks(): - for op in block.operations: - if op.opname == "direct_call": - print op - graph = get_graph(op.args[0], t) - assert graph is None + found = [] + def walkgraph(graph): + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == "direct_call": + print op + subgraph = get_graph(op.args[0], t) + if subgraph is None: + found.append(op) + else: + walkgraph(subgraph) + walkgraph(graph) + assert len(found) == 1 def addone(x): return x + 1 From arigo at codespeak.net Wed Sep 5 18:47:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 18:47:09 +0200 (CEST) Subject: [pypy-svn] r46353 - pypy/dist/pypy/module/thread/test Message-ID: <20070905164709.60F748135@code0.codespeak.net> Author: arigo Date: Wed Sep 5 18:47:08 2007 New Revision: 46353 Modified: pypy/dist/pypy/module/thread/test/support.py pypy/dist/pypy/module/thread/test/test_local.py pypy/dist/pypy/module/thread/test/test_thread.py Log: These tests are quite slow. One of them timed out on wyvern, so I'm adding a way to increase the default timeout again... Modified: pypy/dist/pypy/module/thread/test/support.py ============================================================================== --- pypy/dist/pypy/module/thread/test/support.py (original) +++ pypy/dist/pypy/module/thread/test/support.py Wed Sep 5 18:47:08 2007 @@ -4,10 +4,12 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, interp2app_temp -def waitfor(space, w_condition, timeout=300.0): +NORMAL_TIMEOUT = 300.0 # 5 minutes + +def waitfor(space, w_condition, delay=1): w_sleep = space.appexec([], "():\n import time; return time.sleep") adaptivedelay = 0.04 - limit = time.time() + timeout + limit = time.time() + delay * NORMAL_TIMEOUT while time.time() <= limit: space.call_function(w_sleep, space.wrap(adaptivedelay)) gc.collect() @@ -25,9 +27,9 @@ cls.space = space if option.runappdirect: - def plain_waitfor(condition, timeout=300.0): + def plain_waitfor(condition, delay=1): adaptivedelay = 0.04 - limit = time.time() + timeout + limit = time.time() + NORMAL_TIMEOUT * delay while time.time() <= limit: time.sleep(adaptivedelay) gc.collect() @@ -43,28 +45,3 @@ import time return time.sleep """) - -## cls.w_waitfor = space.appexec([], """(): -## import time -## def waitfor(expr, timeout=10.0): -## limit = time.time() + timeout -## while time.time() <= limit: -## time.sleep(0.002) -## if expr(): -## return -## print '*** timed out ***' -## return waitfor -## """) -## cls.w_busywait = space.appexec([], """(): -## import time -## def busywait(t): -## limit = time.time() + t -## while time.time() <= limit: -## time.sleep(0.002) -## return busywait -## """) - -## space.appexec([], """(): -## import sys -## sys.setcheckinterval(1) -## """) Modified: pypy/dist/pypy/module/thread/test/test_local.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_local.py (original) +++ pypy/dist/pypy/module/thread/test/test_local.py Wed Sep 5 18:47:08 2007 @@ -32,7 +32,7 @@ ok.append(success) for i in range(20): thread.start_new_thread(f, (i,)) - self.waitfor(lambda: len(ok) == 20) #, timeout=30.0) + self.waitfor(lambda: len(ok) == 20, delay=3) assert ok == 20*[True] # see stdout/stderr for failures in the threads self.waitfor(lambda: len(freed) >= 40) @@ -62,7 +62,7 @@ seen.append(x.tag) for i in range(5): thread.start_new_thread(f, ()) - self.waitfor(lambda: len(seen) == 5) #, timeout=20.0) + self.waitfor(lambda: len(seen) == 5, delay=2) seen1 = seen[:] seen1.sort() assert seen1 == [1, 2, 3, 4, 5] @@ -82,5 +82,5 @@ done.append(1) for i in range(5): thread.start_new_thread(f, (i,)) - self.waitfor(lambda: len(done) == 5) #, timeout=20.0) + self.waitfor(lambda: len(done) == 5, delay=2) assert len(done) == 5 Modified: pypy/dist/pypy/module/thread/test/test_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_thread.py Wed Sep 5 18:47:08 2007 @@ -138,6 +138,6 @@ thread.start_new_thread(f, (i, done)) done_marker.append(done) for done in done_marker: - self.waitfor(lambda: done) #, timeout=30.0) + self.waitfor(lambda: done, delay=3) assert done # see stderr for failures in threads assert sorted(lst) == range(120) From arigo at codespeak.net Wed Sep 5 19:14:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 19:14:58 +0200 (CEST) Subject: [pypy-svn] r46354 - in pypy/dist/pypy/rpython: lltypesystem module Message-ID: <20070905171458.497B28139@code0.codespeak.net> Author: arigo Date: Wed Sep 5 19:14:57 2007 New Revision: 46354 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/module/ll_time.py Log: MarkSweepGC.collect() -> time.time() -> wrapper from rffi The latter takes a *arg, which needs to be GC-allocated -> boom Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Sep 5 19:14:57 2007 @@ -28,7 +28,7 @@ def llexternal(name, args, result, _callable=None, sources=[], includes=[], libraries=[], include_dirs=[], sandboxsafe=False, - canraise=False, stringpolicy='noauto'): + canraise=False, stringpolicy='noauto', _nowrapper=False): """ String policies: autocast - automatically cast to ll_string, but don't delete it fullauto - automatically cast + delete is afterwards @@ -52,6 +52,9 @@ if isinstance(_callable, ll2ctypes.LL2CtypesCallable): _callable.funcptr = funcptr + if _nowrapper: + return funcptr + unrolling_arg_tps = unrolling_iterable(enumerate(args)) def wrapper(*args): # XXX the next line is a workaround for the annotation bug Modified: pypy/dist/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_time.py (original) +++ pypy/dist/pypy/rpython/module/ll_time.py Wed Sep 5 19:14:57 2007 @@ -47,25 +47,33 @@ @registering(time.time) def register_time_time(self): + # Note: time.time() is used by the framework GC during collect(), + # which means that we have to be very careful about not allocating + # GC memory here. This is the reason for the _nowrapper=True. + # AWFUL if self.HAVE_GETTIMEOFDAY: if self.GETTIMEOFDAY_NO_TZ: c_gettimeofday = self.llexternal('gettimeofday', - [self.TIMEVALP], rffi.INT) + [self.TIMEVALP], rffi.INT, + _nowrapper=True) else: c_gettimeofday = self.llexternal('gettimeofday', - [self.TIMEVALP, rffi.VOIDP], rffi.INT) + [self.TIMEVALP, rffi.VOIDP], rffi.INT, + _nowrapper=True) else: c_gettimeofday = None if self.HAVE_FTIME: self.configure(CConfigForFTime) c_ftime = self.llexternal('ftime', [lltype.Ptr(self.TIMEB)], - lltype.Void) + lltype.Void, + _nowrapper=True) else: c_ftime = None # to not confuse the flow space - c_time = self.llexternal('time', [rffi.VOIDP], self.TIME_T) + c_time = self.llexternal('time', [rffi.VOIDP], self.TIME_T, + _nowrapper=True) def time_time_llimpl(): void = lltype.nullptr(rffi.VOIDP.TO) From arigo at codespeak.net Wed Sep 5 19:23:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 19:23:43 +0200 (CEST) Subject: [pypy-svn] r46355 - pypy/dist/pypy/rpython/memory/test Message-ID: <20070905172343.DA6AD8122@code0.codespeak.net> Author: arigo Date: Wed Sep 5 19:23:43 2007 New Revision: 46355 Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py Log: Disabling TestMarkSweepGCRunningOnLLinterp. The idea is that the mark-and-sweep GC is already tested differently. The reason for disabling it is that time.time() now involves performing raw mallocs of 'struct timeval', and the old lltype simulator doesn't know about raw mallocs. Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Wed Sep 5 19:23:43 2007 @@ -106,8 +106,9 @@ class TestMarkSweepGC(GCTest): GCClass = MarkSweepGC -class TestMarkSweepGCRunningOnLLinterp(GCTestOnLLInterp): - GCClass = MarkSweepGC +if 0: ## - - - disabling this: the simulator is getting deprecated + class TestMarkSweepGCRunningOnLLinterp(GCTestOnLLInterp): + GCClass = MarkSweepGC class TestSemiSpaceGC(GCTest): GCClass = SemiSpaceGC From arigo at codespeak.net Wed Sep 5 19:42:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Sep 2007 19:42:13 +0200 (CEST) Subject: [pypy-svn] r46356 - pypy/dist/pypy/jit/timeshifter Message-ID: <20070905174213.CA440810C@code0.codespeak.net> Author: arigo Date: Wed Sep 5 19:42:13 2007 New Revision: 46356 Modified: pypy/dist/pypy/jit/timeshifter/hrtyper.py Log: Patching lltype._func objects is now forbidden. This checkin is mostly whacking at the HRTyper source until it doesn't need such patching any more. It's not clean. Modified: pypy/dist/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/hrtyper.py Wed Sep 5 19:42:13 2007 @@ -122,6 +122,8 @@ self.origportalgraph = origportalgraph if origportalgraph: self.portalgraph = bk.get_graph_by_key(origportalgraph, None) + self.portalgraphcopy = flowmodel.copygraph(self.portalgraph, + shallow=True) leaveportalgraph = self.portalgraph else: self.portalgraph = None @@ -175,18 +177,17 @@ flowmodel.checkgraph(graph) base_translator.graphs.append(graph) - # remember a shared pointer for the portal graph, - # so that it can be later patched by rewire_portal. - # this pointer is going to be used by the resuming logic + # return a fnptr to the given tsgraph - with a special case for + # the portal graph, in which case we return a fnptr to the + # portalgraphcopy. The latter will be patched in rewire_portal(). + # this fnptr is going to be used by the resuming logic # and portal (re)entry. def naked_tsfnptr(self, tsgraph): if tsgraph is self.portalgraph: - try: - return self.portal_tsfnptr - except AttributeError: - self.portal_tsfnptr = self.gettscallable(tsgraph) - return self.portal_tsfnptr - return self.gettscallable(tsgraph) + tsgraph2 = self.portalgraphcopy + else: + tsgraph2 = tsgraph + return self.gettscallable(tsgraph, tsgraph2) def rewire_portal(self): origportalgraph = self.origportalgraph @@ -208,13 +209,17 @@ args_specification.append(arg_spec) tsportalgraph = portalgraph - # patch the shared portal pointer - portalgraph = flowmodel.copygraph(tsportalgraph, shallow=True) + # patch the shared portal pointer again + portalgraphcopy2 = flowmodel.copygraph(portalgraph, shallow=True) + portalgraph = self.portalgraphcopy + portalgraph.startblock = portalgraphcopy2.startblock + portalgraph.returnblock = portalgraphcopy2.returnblock + portalgraph.exceptblock = portalgraphcopy2.exceptblock portalgraph.tag = 'portal' self.annotator.translator.graphs.append(portalgraph) portal_fnptr = self.naked_tsfnptr(self.portalgraph) - portal_fnptr._obj.graph = portalgraph + assert portal_fnptr._obj.graph is portalgraph portal_fn = PseudoHighLevelCallable( portal_fnptr, @@ -537,14 +542,14 @@ args_hs, hs_res = self.get_sig_hs(tsgraph) return [self.getrepr(hs_arg) for hs_arg in args_hs] - def gettscallable(self, tsgraph): + def gettscallable(self, tsgraph, tsgraphcopy=None): args_r = self.get_args_r(tsgraph) ARGS = [self.r_JITState.lowleveltype] ARGS += [r.lowleveltype for r in args_r] RESULT = self.r_JITState.lowleveltype return lltype.functionptr(lltype.FuncType(ARGS, RESULT), tsgraph.name, - graph=tsgraph) + graph=tsgraphcopy or tsgraph) def get_timeshift_mapper(self, graph2ts): # XXX try to share the results between "similar enough" graph2ts'es From simonb at codespeak.net Wed Sep 5 20:22:31 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Wed, 5 Sep 2007 20:22:31 +0200 (CEST) Subject: [pypy-svn] r46357 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070905182231.9259A8132@code0.codespeak.net> Author: simonb Date: Wed Sep 5 20:22:30 2007 New Revision: 46357 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: get transpose to work, add a reshape method Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Wed Sep 5 20:22:30 2007 @@ -49,6 +49,15 @@ return SomeObject.getattr(s_array, s_attr) return s + def method_reshape(self, s_tuple): + if not isinstance(s_tuple, SomeTuple): + raise AnnotatorError("reshape expects tuple arg") + for s_item in s_tuple.items: + if not isinstance(s_item, SomeInteger): + raise AnnotatorError("bad shape arg") + ndim = len(s_tuple.items) + return SomeArray(self.typecode, ndim) + def method_transpose(self): return SomeArray(self.typecode, self.ndim) @@ -143,7 +152,6 @@ valid_typecodes='bhilqBHILQfd' class ArrayCallEntry(ExtRegistryEntry): - "Annotation and rtyping of calls to numpy.array" _about_ = numpy.array def compute_result_annotation(self, s_list, s_dtype=None): @@ -178,7 +186,6 @@ class EmptyCallEntry(ExtRegistryEntry): - "Annotation and rtyping of calls to numpy.empty" _about_ = numpy.empty def compute_result_annotation(self, s_arg, s_dtype=None): @@ -191,7 +198,7 @@ elif isinstance(s_arg, SomeInteger): ndim = 1 else: - raise AnnotatorError("shape arg not understood") + raise AnnotatorError("shape arg not understood: %s"%s_arg) typecode = 'd' if isinstance(s_dtype, SomeChar) and s_dtype.is_constant(): @@ -208,7 +215,6 @@ return v_result class ZerosCallEntry(EmptyCallEntry): - "Annotation and rtyping of calls to numpy.zeros" _about_ = numpy.zeros def specialize_call(self, hop, i_dtype=None): Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Wed Sep 5 20:22:30 2007 @@ -34,6 +34,22 @@ return array return ll_build_from_shape +def gen_build_alias_shape(ndim): + unrolling_dims = unrolling_iterable(reversed(range(ndim))) + def ll_build_alias_shape(ARRAY, ao, shape): + array = ll_allocate(ARRAY, ndim) + itemsize = 1 + for i in unrolling_dims: + attr = 'item%d'%i + size = getattr(shape, attr) + array.shape[i] = size + array.strides[i] = itemsize + itemsize *= size + array.data = ao.data + array.dataptr = ao.dataptr + return array + return ll_build_alias_shape + def gen_get_shape(ndim): unrolling_dims = unrolling_iterable(range(ndim)) def ll_get_shape(ARRAY, TUPLE, array): @@ -199,7 +215,6 @@ ndim = self.s_array.ndim if isinstance(r_arg, TupleRepr): r_tuple, v_tuple = r_arg, v_arg - cTUPLE = inputconst(lltype.Void, r_tuple.lowleveltype.TO) ll_build_from_shape = gen_build_from_shape(ndim, zero) c_ndim = inputconst(lltype.Signed, ndim) assert ndim == len(r_tuple.items_r) @@ -216,6 +231,17 @@ cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) return hop.gendirectcall(ll_transpose, cARRAY, v_self) + def rtype_method_reshape(self, hop): + r_result = hop.r_result + r_tuple = hop.args_r[1] + if not isinstance(r_tuple, TupleRepr): + raise TyperError() + ndim = len(r_tuple.items_r) + ll_build_alias_shape = gen_build_alias_shape(ndim) + [v_array, v_tuple] = hop.inputargs(self, r_tuple) + cARRAY = inputconst(lltype.Void, r_result.lowleveltype.TO) + return hop.llops.gendirectcall(ll_build_alias_shape, cARRAY, v_array, v_tuple) + def get_ndim(self, hop, v_array): cname = inputconst(Void, 'ndim') return hop.llops.genop('getfield', [v_array, cname], resulttype=Signed) @@ -437,10 +463,15 @@ array.dataptr = direct_arrayitems(array.data) return array -def ll_transpose(ARRAY, a1): - a2 = ll_build_alias(ARRAY, a1) - # XX do something to a2 - return a2 +def ll_transpose(ARRAY, ao): + ndim = ao.ndim + array = ll_allocate(ARRAY, ndim) + array.data = ao.data # alias data + for i in range(ndim): + array.shape[i] = ao.shape[ndim-i-1] + array.strides[i] = ao.strides[ndim-i-1] + array.dataptr = ao.dataptr + return array Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Wed Sep 5 20:22:30 2007 @@ -148,24 +148,33 @@ assert s.typecode == 'd' def test_annotate_array_attr(self): - def f(): + def fget(): a1 = numpy.array([1,2]) return a1.shape t = TranslationContext() a = t.buildannotator() - s = a.build_types(f, []) + s = a.build_types(fget, []) assert type(s) == SomeTuple def test_annotate_array_method(self): - def f(): - a1 = numpy.array([1,2]) - return a1.transpose() + def f_transpose(): + a = numpy.zeros((3,4)) + return a.transpose() t = TranslationContext() a = t.buildannotator() - s = a.build_types(f, []) + s = a.build_types(f_transpose, []) + assert type(s) == SomeArray + assert s.ndim == 2 + + def f_reshape(): + a = numpy.array(range(12)) + return a.reshape((3,4)) + + s = a.build_types(f_reshape, []) assert type(s) == SomeArray + assert s.ndim == 2 def test_annotate_indexing(self): def f(): @@ -284,13 +293,27 @@ #assert len(res.data) == 3*4*5 # GcArray has nolength def test_specialize_array_method(self): - def f(): - a = numpy.array([1,2]) + def f_transpose(): + a = numpy.zeros((3,4)) return a.transpose() - res = interpret(f, []) - assert res.data[0] == 1 - assert res.data[1] == 2 + res = interpret(f_transpose, []) + assert res.shape[0] == 4 + assert res.shape[1] == 3 + + def f_reshape(): + a = numpy.array(range(12)) + b = a.reshape((3,4)) + b[1,2] = 0 + return b + + res = interpret(f_reshape, []) + assert res.shape[0] == 3 + assert res.shape[1] == 4 + assert res.strides[0] == 4 + assert res.strides[1] == 1 + assert res.dataptr[5] == 5 + assert res.dataptr[6] == 0 def test_specialize_view_0(self): def f(): From antocuni at codespeak.net Wed Sep 5 20:43:54 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 Sep 2007 20:43:54 +0200 (CEST) Subject: [pypy-svn] r46358 - pypy/dist/pypy/translator/cli/test Message-ID: <20070905184354.E78998123@code0.codespeak.net> Author: antocuni Date: Wed Sep 5 20:43:54 2007 New Revision: 46358 Modified: pypy/dist/pypy/translator/cli/test/test_carbonpython.py Log: skip this test for now. I never managed to reproduce it on my machine or wyvern, no clue why it fails sometimes :-(. Modified: pypy/dist/pypy/translator/cli/test/test_carbonpython.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_carbonpython.py (original) +++ pypy/dist/pypy/translator/cli/test/test_carbonpython.py Wed Sep 5 20:43:54 2007 @@ -105,6 +105,7 @@ assert entrypoints[1][1] == (MyClass, int) # foo inputtypes def test_compile_class(self): + py.test.skip('This test fails every other day. No clue why :-(') class MyClass: @export(int) def __init__(self, x): From antocuni at codespeak.net Wed Sep 5 21:10:46 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 Sep 2007 21:10:46 +0200 (CEST) Subject: [pypy-svn] r46359 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070905191046.2BB4D8121@code0.codespeak.net> Author: antocuni Date: Wed Sep 5 21:10:44 2007 New Revision: 46359 Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Log: the old implementation of ulong_to_double was broken. This one is also broken, but only for very large values. Not sure what's the best way to implement ullong_* operations on JVM :-(. Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Wed Sep 5 21:10:44 2007 @@ -22,7 +22,7 @@ public static final long LONG_MIN = Long.MIN_VALUE; public static final int INT_MAX = Integer.MAX_VALUE; public static final int INT_MIN = Integer.MIN_VALUE; - + public static final double ULONG_MAX = 18446744073709551616.0; /** * Compares two unsigned integers (value1 and value2) and returns @@ -130,14 +130,13 @@ return (double)uint_to_long(value); } + + // XXX: broken if the value is too large public static double ulong_to_double(long value) { if (value >= 0) return value; else { - long lopart = value & 0xFFFFFFFF; - long hipart = value >>> 32; - double result = (hipart << 32) | lopart; - return result; + return ULONG_MAX + value; } } @@ -952,7 +951,7 @@ ensure(ulong_cmp(0xFFFF, 0xFFFFFFFFFFFFFFFFL) < 0); ensure(ulong_cmp(0x8000000000000000L, 0xFFFF) > 0); ensure(ulong_cmp(0xFFFF, 0x8000000000000000L) < 0); - + System.out.println("Total Failures: "+__failures); } } From antocuni at codespeak.net Wed Sep 5 22:02:37 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 Sep 2007 22:02:37 +0200 (CEST) Subject: [pypy-svn] r46360 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20070905200237.28006810D@code0.codespeak.net> Author: antocuni Date: Wed Sep 5 22:02:35 2007 New Revision: 46360 Modified: pypy/dist/pypy/translator/jvm/metavm.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/test/test_cast.py Log: implement cast_primitive for jvm Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Wed Sep 5 22:02:35 2007 @@ -1,3 +1,4 @@ +from pypy.rpython.ootypesystem import ootype from pypy.translator.oosupport.metavm import MicroInstruction from pypy.translator.jvm.typesystem import JvmScalarType, JvmClassType import pypy.translator.jvm.generator as jvmgen @@ -130,3 +131,19 @@ generator.cast_weak_address_to_ptr(RESULTTYPE) CastWeakAddressToPtr = _CastWeakAddressToPtr() + +CASTS = { +# FROM TO + (ootype.Signed, ootype.UnsignedLongLong): jvmgen.I2L, + (ootype.SignedLongLong, ootype.Signed): jvmgen.L2I, + (ootype.UnsignedLongLong, ootype.SignedLongLong): None, + } + +class _CastPrimitive(MicroInstruction): + def render(self, generator, op): + FROM = op.args[0].concretetype + TO = op.result.concretetype + opcode = CASTS[(FROM, TO)] + if opcode: + generator.emit(opcode) +CastPrimitive = _CastPrimitive() Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Wed Sep 5 22:02:35 2007 @@ -10,7 +10,7 @@ SetField, GetField, DownCast, RuntimeNew, OOString, CastTo, PushPrimitive from pypy.translator.jvm.metavm import \ IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ - CastPtrToWeakAddress, CastWeakAddressToPtr + CastPtrToWeakAddress, CastWeakAddressToPtr, CastPrimitive from pypy.rpython.ootypesystem import ootype import pypy.translator.jvm.generator as jvmgen @@ -243,6 +243,7 @@ 'cast_float_to_uint': jvmgen.PYPYDOUBLETOUINT, 'truncate_longlong_to_int': jvmgen.L2I, 'cast_longlong_to_float': jvmgen.L2D, + 'cast_primitive': [PushAllArgs, CastPrimitive, StoreResult], 'is_early_constant': [PushPrimitive(ootype.Bool, False), StoreResult] } Modified: pypy/dist/pypy/translator/jvm/test/test_cast.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_cast.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_cast.py Wed Sep 5 22:02:35 2007 @@ -3,6 +3,12 @@ from pypy.translator.oosupport.test_template.cast import BaseTestCast class TestCast(BaseTestCast, JvmTest): - def test_cast_primitive(self): - py.test.skip('fixme!') + # genjvm has buggy support for ullong, so the original test + # doesn't work + from pypy.rpython.lltypesystem.lltype import cast_primitive, UnsignedLongLong + def f(x): + x = cast_primitive(UnsignedLongLong, x) + return x + res = self.interpret(f, [14]) + assert res == 14 From niko at codespeak.net Thu Sep 6 06:24:25 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 6 Sep 2007 06:24:25 +0200 (CEST) Subject: [pypy-svn] r46361 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20070906042425.0BFE1810F@code0.codespeak.net> Author: niko Date: Thu Sep 6 06:24:24 2007 New Revision: 46361 Modified: pypy/dist/pypy/translator/jvm/constant.py pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/test/test_constant.py pypy/dist/pypy/translator/jvm/typesystem.py Log: modify jvm constant generation so that it spreads the constants out over several classes, rather than grouping all of the static fields together into one class. This avoids overflowing the constant pool when there are too many constants, but doens't fix test_manyconstant, which has a single list with 7500 elements, and simply generates too much code. Modified: pypy/dist/pypy/translator/jvm/constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/constant.py (original) +++ pypy/dist/pypy/translator/jvm/constant.py Thu Sep 6 06:24:24 2007 @@ -4,28 +4,55 @@ Field, Method, CUSTOMDICTMAKE from pypy.translator.oosupport.constant import \ BaseConstantGenerator, RecordConst, InstanceConst, ClassConst, \ - StaticMethodConst, CustomDictConst, WeakRefConst, push_constant + StaticMethodConst, CustomDictConst, WeakRefConst, push_constant, \ + MAX_CONST_PER_STEP from pypy.translator.jvm.typesystem import \ - jPyPyConst, jObject, jVoid, jWeakRef + jObject, jVoid, jWeakRef, JvmClassType + +jPyPyConstantInit = JvmClassType('pypy.ConstantInit') +jPyPyConstantInitMethod = Method.s(jPyPyConstantInit, 'init', [], jVoid) # ___________________________________________________________________________ # Constant Generator class JVMConstantGenerator(BaseConstantGenerator): - + + def __init__(self, db): + BaseConstantGenerator.__init__(self, db) + self.num_constants = 0 + self.ccs = [] + + def runtime_init(self, gen): + """ + Called from node.EntryPoint to generate code that initializes + all of the constants. Right now, this invokes a known static + method, but this should probably be changed eventually. + """ + gen.emit(jPyPyConstantInitMethod) + # _________________________________________________________________ # Constant Operations # - # We store constants in static fields of the jPyPyConst class. + # We store constants in static fields of a constant class; we + # generate new constant classes every MAX_CONST_PER_STEP constants + # to avoid any particular class getting too big. def _init_constant(self, const): # Determine the Java type of the constant: some constants # (weakrefs) do not have an OOTYPE, so if it returns None use - # jtype() + # jtype() JFIELDOOTY = const.OOTYPE() if not JFIELDOOTY: jfieldty = const.jtype() else: jfieldty = self.db.lltype_to_cts(JFIELDOOTY) - const.fieldobj = Field(jPyPyConst.name, const.name, jfieldty, True) + + # Every MAX_CONST_PER_STEP constants, we create a new class. + # This prevents any one class from getting too big. + if (self.num_constants % MAX_CONST_PER_STEP) == 0: + cc_num = len(self.ccs) + self.ccs.append(JvmClassType('pypy.Constant_%d' % cc_num)) + self.num_constants += 1 + + const.fieldobj = Field(self.ccs[-1].name, const.name, jfieldty, True) def push_constant(self, gen, const): const.fieldobj.load(gen) @@ -35,27 +62,71 @@ # _________________________________________________________________ # Constant Generation + # + # The JVM constants are generated as follows: + # + # First, a set of classes are used as simple structs with static + # fields that store each constant. These class names have already + # been generated, and they are stored in the member array self.ccs. + # Therefore, the first thing we do is to generate these classes + # by iterating over all constants and declaring their fields. + # + # We then generate initialization code the constants in a SEPARATE + # set of classes, named pypy.ConstantInit_NNN. We generate one such + # class for each "step" of the underlying BaseConstantGenerator. + # + # Note that, in this setup, we cannot rely on the JVM's class init + # to initialize our constants for us: instead, we generate a static + # method (jPyPyConstantInitMethod) in _end_gen_constants() that + # invokes each of the ConstantInit_NNN's methods. + # + # Normally, these static field classes and the initialization + # code are stored together. The JVM stores them seperately, + # because otherwise it is quite hard to ensure that (a) the + # constants are initialized in the right order, and (b) all of + # the constant declarations are emitted when they are needed. + + def gen_constants(self, ilasm): + self.step_classes = [] + + # First, create the classes that store the static fields. + constants_by_cls = {} + for const in self.cache.values(): + try: + constants_by_cls[const.fieldobj.class_name].append(const) + except KeyError: + constants_by_cls[const.fieldobj.class_name] = [const] + for cc in self.ccs: + ilasm.begin_class(cc, jObject) + for const in constants_by_cls[cc.name]: + ilasm.add_field(const.fieldobj) + ilasm.end_class() + + # Now, delegate to the normal code for the rest + super(JVMConstantGenerator, self).gen_constants(ilasm) def _begin_gen_constants(self, gen, all_constants): - gen.begin_class(jPyPyConst, jObject) return gen def _declare_const(self, gen, const): - gen.add_field(const.fieldobj) + # in JVM, this is done first, in gen_constants() + return def _declare_step(self, gen, stepnum): - next_nm = "constant_init_%d" % stepnum - gen.begin_function(next_nm, [], [], jVoid, True) + self.step_classes.append(JvmClassType('pypy.ConstantInit_%d' % stepnum)) + gen.begin_class(self.step_classes[-1], jObject) + gen.begin_function('constant_init', [], [], jVoid, True) def _close_step(self, gen, stepnum): gen.return_val(jVoid) - gen.end_function() # end constant_init_N where N == stepnum + gen.end_function() # end constant_init() + gen.end_class() # end pypy.ConstantInit_NNN def _end_gen_constants(self, gen, numsteps): - # The static init code just needs to call constant_init_1..N - gen.begin_function('', [], [], jVoid, True) - for x in range(numsteps): - m = Method.s(jPyPyConst, "constant_init_%d" % x, [], jVoid) + gen.begin_class(jPyPyConstantInit, jObject) + gen.begin_j_function(jPyPyConstantInit, jPyPyConstantInitMethod) + for cls in self.step_classes: + m = Method.s(cls, "constant_init", [], jVoid) gen.emit(m) gen.return_val(jVoid) gen.end_function() Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Thu Sep 6 06:24:24 2007 @@ -13,7 +13,7 @@ from pypy.translator.jvm.generator import Method, Property, Field import pypy.translator.jvm.constant as jvmconst from pypy.translator.jvm.typesystem import \ - jStringBuilder, jInt, jVoid, jString, jChar, jPyPyConst, jObject, \ + jStringBuilder, jInt, jVoid, jString, jChar, jObject, \ jThrowable, JvmNativeClass from pypy.translator.jvm.builtin import JvmBuiltInType from pypy.rpython.lltypesystem.llmemory import WeakGcAddress Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Thu Sep 6 06:24:24 2007 @@ -624,9 +624,11 @@ appropriate arguments to define a method on class 'cls_obj' that could be invoked with 'method_obj'. """ + if method_obj.is_static(): def_args = [] + else: def_args = [cls_obj] return self.begin_function(method_obj.method_name, [], - [cls_obj]+method_obj.argument_types, + def_args+method_obj.argument_types, method_obj.return_type, static=method_obj.is_static(), abstract=abstract) @@ -1263,7 +1265,7 @@ class JasminGenerator(JVMGenerator): - def __init__(self, db, outdir, package): + def __init__(self, db, outdir): JVMGenerator.__init__(self, db) self.outdir = outdir @@ -1304,7 +1306,8 @@ assert self.curclass is None def add_comment(self, comment): - self.curclass.out(" ; %s\n" % comment) + if self.curclass: + self.curclass.out(" ; %s\n" % comment) def implements(self, jinterface): self.curclass.out( Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Thu Sep 6 06:24:24 2007 @@ -269,7 +269,6 @@ configuration. Right now, however, there is only one kind of generator: JasminGenerator """ print "Uh...?" - return JasminGenerator( - self.db, self.jvmsrc.javadir, self.jvmsrc.package) + return JasminGenerator(self.db, self.jvmsrc.javadir) Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Thu Sep 6 06:24:24 2007 @@ -94,11 +94,17 @@ gen.begin_function( 'main', (), [jStringArray], jVoid, static=True) - # First thing we do is setup the PyPy helper. For now this is - # a static variable of the PyPy class, though that precludes - # running multiple translations. + # Initialization: + # + # 1. Setup the PyPy helper, which (for now) is a static + # variable of the PyPy class, though that precludes running + # multiple translations. + # + # 2. Run the initialization method for the constant class. + # gen.new_with_jtype(gen.db.interlink_class) jvmgen.PYPYINTERLINK.store(gen) + gen.db.constant_generator.runtime_init(gen) if self.print_result: gen.begin_try() Modified: pypy/dist/pypy/translator/jvm/test/test_constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_constant.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_constant.py Thu Sep 6 06:24:24 2007 @@ -131,7 +131,7 @@ assert self.interpret(fn, [0, 0]) == 3 def test_many_constants(self): - py.test.skip('fixme!') + py.test.skip('Initializing a large constant list generates a function too large for the JVM limits') N = 7500 class A: pass Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Thu Sep 6 06:24:24 2007 @@ -173,7 +173,6 @@ jWeakRef = JvmClassType('java.lang.ref.WeakReference') jPyPy = JvmClassType('pypy.PyPy') jPyPyExcWrap = JvmClassType('pypy.ExceptionWrapper') -jPyPyConst = JvmClassType('pypy.Constant') jPyPyMain = JvmClassType('pypy.Main') jPyPyDictItemsIterator = JvmClassType('pypy.DictItemsIterator') jPyPyInterlink = JvmClassType('pypy.Interlink') From arigo at codespeak.net Thu Sep 6 10:05:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 6 Sep 2007 10:05:04 +0200 (CEST) Subject: [pypy-svn] r46362 - pypy/dist/pypy/module/thread/test Message-ID: <20070906080504.314CB8102@code0.codespeak.net> Author: arigo Date: Thu Sep 6 10:05:02 2007 New Revision: 46362 Modified: pypy/dist/pypy/module/thread/test/support.py Log: The app-level time.sleep() doesn't release the GIL, and there is little chance that we'll fix it when running on top of py.py. Simple workaround. Modified: pypy/dist/pypy/module/thread/test/support.py ============================================================================== --- pypy/dist/pypy/module/thread/test/support.py (original) +++ pypy/dist/pypy/module/thread/test/support.py Thu Sep 6 10:05:02 2007 @@ -7,11 +7,13 @@ NORMAL_TIMEOUT = 300.0 # 5 minutes def waitfor(space, w_condition, delay=1): - w_sleep = space.appexec([], "():\n import time; return time.sleep") adaptivedelay = 0.04 limit = time.time() + delay * NORMAL_TIMEOUT while time.time() <= limit: - space.call_function(w_sleep, space.wrap(adaptivedelay)) + GIL = space.threadlocals.GIL + GIL.release() + time.sleep(adaptivedelay) + GIL.acquire(True) gc.collect() if space.is_true(space.call_function(w_condition)): return From antocuni at codespeak.net Thu Sep 6 10:53:02 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 Sep 2007 10:53:02 +0200 (CEST) Subject: [pypy-svn] r46364 - pypy/dist/pypy/translator/cli/test Message-ID: <20070906085302.0F7C08136@code0.codespeak.net> Author: antocuni Date: Thu Sep 6 10:53:02 2007 New Revision: 46364 Modified: pypy/dist/pypy/translator/cli/test/test_carbonpython.py Log: missing import Modified: pypy/dist/pypy/translator/cli/test/test_carbonpython.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_carbonpython.py (original) +++ pypy/dist/pypy/translator/cli/test/test_carbonpython.py Thu Sep 6 10:53:02 2007 @@ -1,3 +1,4 @@ +import py import os import os.path from pypy.tool import udir From antocuni at codespeak.net Thu Sep 6 14:18:07 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 Sep 2007 14:18:07 +0200 (CEST) Subject: [pypy-svn] r46365 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20070906121807.2197E80F7@code0.codespeak.net> Author: antocuni Date: Thu Sep 6 14:18:05 2007 New Revision: 46365 Modified: pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/jvm/metavm.py pypy/dist/pypy/translator/jvm/methods.py pypy/dist/pypy/translator/jvm/test/test_class.py Log: take care of void arguments Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Thu Sep 6 14:18:05 2007 @@ -268,7 +268,6 @@ """ Creates and returns a Generator object according to the configuration. Right now, however, there is only one kind of generator: JasminGenerator """ - print "Uh...?" return JasminGenerator(self.db, self.jvmsrc.javadir) Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Thu Sep 6 14:18:05 2007 @@ -14,6 +14,7 @@ class _JvmCallMethod(MicroInstruction): def _invoke_method(self, gen, db, jmethod, jactargs, args, jactres, res): + assert len(args) == len(jactargs) for arg, jmthdty in zip(args, jactargs): # Load the argument on the stack: gen.load(arg) @@ -44,11 +45,12 @@ jactargs = jmethod.argument_types if jmethod.is_static(): jactargs = jactargs[1:] - + + args = [arg for arg in op.args[2:] if arg.concretetype is not ootype.Void] # Iterate through the arguments, inserting casts etc as required gen.load(this) self._invoke_method(gen, gen.db, jmethod, - jactargs, op.args[2:], + jactargs, args, jmethod.return_type, op.result) JvmCallMethod = _JvmCallMethod() Modified: pypy/dist/pypy/translator/jvm/methods.py ============================================================================== --- pypy/dist/pypy/translator/jvm/methods.py (original) +++ pypy/dist/pypy/translator/jvm/methods.py Thu Sep 6 14:18:05 2007 @@ -61,11 +61,8 @@ for fieldnm, (FIELDOOTY, fielddef) in self.OOCLASS._fields.iteritems(): if FIELDOOTY is ootype.Void: continue - genprint('"'+fieldnm+'":') - print "fieldnm=%r fieldty=%r" % (fieldnm, FIELDOOTY) - # Print the value of the field: self._print_field_value(fieldnm, FIELDOOTY) Modified: pypy/dist/pypy/translator/jvm/test/test_class.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_class.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_class.py Thu Sep 6 14:18:05 2007 @@ -6,5 +6,25 @@ def test_overridden_classattr_as_defaults(self): py.test.skip("JVM doesn't support overridden default value yet") + def test_method_void_arg(self): + class Space: + def __init__(self): + self.x = 40 + def _freeze_(self): + return True + space = Space() + + class MyClass: + def foo(self, space, x): + return space.x + x + + def fn(x): + obj = MyClass() + return obj.foo(space, x) + + assert self.interpret(fn, [2]) == 42 + + + class TestJvmSpecialCase(JvmTest, BaseTestSpecialcase): pass From antocuni at codespeak.net Thu Sep 6 14:35:43 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 Sep 2007 14:35:43 +0200 (CEST) Subject: [pypy-svn] r46366 - pypy/dist/pypy/translator/jvm Message-ID: <20070906123543.6A94280F9@code0.codespeak.net> Author: antocuni Date: Thu Sep 6 14:35:41 2007 New Revision: 46366 Modified: pypy/dist/pypy/translator/jvm/metavm.py Log: if we are calling a builtin method, we can't drop Void values Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Thu Sep 6 14:35:41 2007 @@ -3,6 +3,7 @@ from pypy.translator.jvm.typesystem import JvmScalarType, JvmClassType import pypy.translator.jvm.generator as jvmgen import pypy.translator.jvm.typesystem as jvmtype +from pypy.translator.jvm.builtin import JvmBuiltInType class _IndirectCall(MicroInstruction): def render(self, gen, op): @@ -40,13 +41,20 @@ thisjtype = gen.db.lltype_to_cts(this.concretetype) jmethod = thisjtype.lookup_method(method.value) + # if this is a builtin-type, the signature is exact and we + # need to keep Void values; else, the signature does not + # include Void values, so we need to drop them. + if isinstance(thisjtype, JvmBuiltInType): + args = op.args[2:] + else: + args = [arg for arg in op.args[2:] if arg.concretetype is not ootype.Void] + # Ugly: if jmethod ends up being a static method, then # peel off the first argument jactargs = jmethod.argument_types if jmethod.is_static(): jactargs = jactargs[1:] - - args = [arg for arg in op.args[2:] if arg.concretetype is not ootype.Void] + # Iterate through the arguments, inserting casts etc as required gen.load(this) self._invoke_method(gen, gen.db, jmethod, From arigo at codespeak.net Thu Sep 6 15:05:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 6 Sep 2007 15:05:48 +0200 (CEST) Subject: [pypy-svn] r46367 - in pypy/dist/pypy/rlib: . test Message-ID: <20070906130548.8822980F3@code0.codespeak.net> Author: arigo Date: Thu Sep 6 15:05:47 2007 New Revision: 46367 Modified: pypy/dist/pypy/rlib/rmarshal.py pypy/dist/pypy/rlib/test/test_rmarshal.py Log: Marshal/unmarshal floats. Modified: pypy/dist/pypy/rlib/rmarshal.py ============================================================================== --- pypy/dist/pypy/rlib/rmarshal.py (original) +++ pypy/dist/pypy/rlib/rmarshal.py Thu Sep 6 15:05:47 2007 @@ -7,6 +7,7 @@ from pypy.annotation.listdef import ListDef, TooLateForChange from pypy.annotation.pairtype import pair, pairtype from pypy.rlib.rarithmetic import formatd, r_longlong, intmask +from pypy.rlib.rarithmetic import break_up_float, parts_to_float from pypy.rlib.unroll import unrolling_iterable class CannotMarshal(Exception): @@ -191,11 +192,11 @@ add_dumper(annmodel.SomeFloat(), dump_float) def load_float(loader): - if loader.readchr() != TYPE_FLOAT: + if readchr(loader) != TYPE_FLOAT: raise ValueError("expected a float") - length = ord(loader.readchr()) - s = loader.read(length) - return xxx # ...mess... + length = ord(readchr(loader)) + s = readstr(loader, length) + return parts_to_float(*break_up_float(s)) add_loader(annmodel.SomeFloat(), load_float) def dump_string_or_none(buf, x): Modified: pypy/dist/pypy/rlib/test/test_rmarshal.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmarshal.py (original) +++ pypy/dist/pypy/rlib/test/test_rmarshal.py Thu Sep 6 15:05:47 2007 @@ -2,6 +2,7 @@ import marshal from pypy.rlib.rmarshal import * from pypy.annotation import model as annmodel +from pypy.rlib.rarithmetic import formatd types_that_can_be_none = [ [int], @@ -53,8 +54,8 @@ buf = 'i\x05\x00\x00\x00' assert get_unmarshaller(int)(buf) == 5 -## buf = 'f\x043.25' -## assert get_unmarshaller(float)(buf) == 3.25 + buf = 'f\x043.25' + assert get_unmarshaller(float)(buf) == 3.25 buf = 's\x0c\x00\x00\x00hello, world' assert get_unmarshaller(str)(buf) == "hello, world" @@ -90,31 +91,33 @@ def test_llinterp_marshal(): from pypy.rpython.test.test_llinterp import interpret - marshaller = get_marshaller([(int, str)]) + marshaller = get_marshaller([(int, str, float)]) def f(): buf = [] - marshaller(buf, [(5, "hello"), (7, "world")]) + marshaller(buf, [(5, "hello", -0.5), (7, "world", 1E100)]) return ''.join(buf) res = interpret(f, []) res = ''.join(res.chars) - assert res == ('[\x02\x00\x00\x00(\x02\x00\x00\x00i\x05\x00\x00\x00' - 's\x05\x00\x00\x00hello(\x02\x00\x00\x00i\x07\x00\x00\x00' - 's\x05\x00\x00\x00world') + assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00' + 's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00' + 'i\x07\x00\x00\x00s\x05\x00\x00\x00world' + 'f\x061e+100') def test_llinterp_unmarshal(): from pypy.rpython.test.test_llinterp import interpret - unmarshaller = get_unmarshaller([(int, str)]) - buf = ('[\x02\x00\x00\x00(\x02\x00\x00\x00i\x05\x00\x00\x00' - 's\x05\x00\x00\x00hello(\x02\x00\x00\x00i\x07\x00\x00\x00' - 's\x05\x00\x00\x00world') + unmarshaller = get_unmarshaller([(int, str, float)]) + buf = ('[\x02\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00' + 's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00' + 'i\x07\x00\x00\x00s\x05\x00\x00\x00world' + 'f\x061e+100') def f(): result = '' - for num, string in unmarshaller(buf): - result += '%d=%s;' % (num, string) + for num, string, fval in unmarshaller(buf): + result += '%d=%s/%s;' % (num, string, formatd('%.17g', fval)) return result res = interpret(f, []) res = ''.join(res.chars) - assert res == '5=hello;7=world;' + assert res == '5=hello/-0.5;7=world/1e+100;' def test_stat_result(): import os From arigo at codespeak.net Thu Sep 6 15:12:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 6 Sep 2007 15:12:12 +0200 (CEST) Subject: [pypy-svn] r46368 - pypy/dist/pypy/translator/sandbox/test Message-ID: <20070906131212.64DE080F3@code0.codespeak.net> Author: arigo Date: Thu Sep 6 15:12:11 2007 New Revision: 46368 Modified: pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Log: A test requiring float unmarshalling. Rename some other tests. Modified: pypy/dist/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Thu Sep 6 15:12:11 2007 @@ -1,5 +1,5 @@ import py -import sys, os +import sys, os, time import struct from pypy.rpython.lltypesystem import rffi @@ -19,7 +19,7 @@ write_message(g, result, resulttype) -def test_sandbox_1(): +def test_open_dup(): def entry_point(argv): fd = os.open("/tmp/foobar", os.O_RDONLY, 0777) assert fd == 77 @@ -37,7 +37,7 @@ f.close() assert tail == "" -def test_sandbox_2(): +def test_read_write(): def entry_point(argv): fd = os.open("/tmp/foobar", os.O_RDONLY, 0777) assert fd == 77 @@ -60,7 +60,7 @@ f.close() assert tail == "" -def test_sandbox_3(): +def test_dup2_access(): def entry_point(argv): os.dup2(34, 56) y = os.access("spam", 77) @@ -76,7 +76,7 @@ f.close() assert tail == "" -def test_sandbox_4(): +def test_stat_ftruncate(): from pypy.rpython.module.ll_os_stat import s_StatResult from pypy.rlib.rarithmetic import r_longlong r0x12380000007 = r_longlong(0x12380000007) @@ -98,6 +98,22 @@ f.close() assert tail == "" +def test_time(): + def entry_point(argv): + t = time.time() + os.dup(int(t*1000)) + return 0 + + t = Translation(entry_point, backend='c', standalone=True, sandbox=True) + exe = t.compile() + g, f = os.popen2(exe, "t", 0) + expect(f, g, "ll_time.ll_time_time", (), 3.141592) + expect(f, g, "ll_os.ll_os_dup", (3141,), 3) + g.close() + tail = f.read() + f.close() + assert tail == "" + def test_oserror(): def entry_point(argv): try: From arigo at codespeak.net Thu Sep 6 17:38:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 6 Sep 2007 17:38:20 +0200 (CEST) Subject: [pypy-svn] r46371 - in pypy/dist/pypy/interpreter: . test Message-ID: <20070906153820.A33D080FE@code0.codespeak.net> Author: arigo Date: Thu Sep 6 17:38:20 2007 New Revision: 46371 Modified: pypy/dist/pypy/interpreter/callmethod.py pypy/dist/pypy/interpreter/test/test_callmethod.py Log: An optimized version of space.call_method() based on the same principle as the CALL_METHOD bytecode. Modified: pypy/dist/pypy/interpreter/callmethod.py ============================================================================== --- pypy/dist/pypy/interpreter/callmethod.py (original) +++ pypy/dist/pypy/interpreter/callmethod.py Thu Sep 6 17:38:20 2007 @@ -13,6 +13,7 @@ from pypy.interpreter import pyframe, function from pypy.rlib.jit import we_are_jitted from pypy.interpreter.argument import Arguments +from pypy.objspace.std import StdObjSpace def object_getattribute(space): @@ -68,3 +69,23 @@ finally: f.dropvalues(nargs + 2) f.pushvalue(w_result) + + +def call_method(space, w_obj, methname, *arg_w): + """An optimized version of space.call_method() + based on the same principle as above. + """ + w_name = space.wrap(methname) + w_getattribute = space.lookup(w_obj, '__getattribute__') + if w_getattribute is object_getattribute(space): + w_descr = space.lookup(w_obj, methname) + if type(w_descr) is function.Function: + w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + if w_value is None: + # fast method path: a function object in the class, + # nothing in the instance + return space.call_function(w_descr, w_obj, *arg_w) + w_meth = space.getattr(w_obj, w_name) + return space.call_function(w_meth, *arg_w) + +StdObjSpace.call_method = call_method Modified: pypy/dist/pypy/interpreter/test/test_callmethod.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_callmethod.py (original) +++ pypy/dist/pypy/interpreter/test/test_callmethod.py Thu Sep 6 17:38:20 2007 @@ -103,3 +103,21 @@ else: raise Exception("did not raise?") """ + + +class TestCallMethod: + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.opcodes.CALL_METHOD": True}) + + def test_space_call_method(self): + space = self.space + w_lst = space.newlist([]) + space.call_method(w_lst, 'append', space.w_False) + res = space.int_w(space.call_method(w_lst, '__len__')) + assert res == 1 + + def test_fallback_case(self): + space = self.space + space.int_w(space.call_method(space.wrap(space.sys), + 'getrecursionlimit')) From arigo at codespeak.net Thu Sep 6 18:04:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 6 Sep 2007 18:04:08 +0200 (CEST) Subject: [pypy-svn] r46374 - pypy/dist/pypy/doc/config Message-ID: <20070906160408.8C3AC80F1@code0.codespeak.net> Author: arigo Date: Thu Sep 6 18:04:07 2007 New Revision: 46374 Modified: pypy/dist/pypy/doc/config/translation.sandbox.txt Log: More docs about --sandbox. Modified: pypy/dist/pypy/doc/config/translation.sandbox.txt ============================================================================== --- pypy/dist/pypy/doc/config/translation.sandbox.txt (original) +++ pypy/dist/pypy/doc/config/translation.sandbox.txt Thu Sep 6 18:04:07 2007 @@ -1,2 +1,15 @@ Generate a special fully-sandboxed executable. -Experimental. + +The fully-sandboxed executable cannot be run directly, but +only as a subprocess of an outer "controlling" process. The +sandboxed process is "safe" in the sense that it doesn't do +any library or system call - instead, whenever it would like +to perform such an operation, it marshals the operation name +and the arguments to its stdout and it waits for the +marshalled result on its stdin. This controller process must +handle these operation requests, in any way it likes, allowing +full virtualization. + +For examples of controller process see +``pypy/translator/sandbox/interact.py`` and +``pypy/translator/sandbox/pypy_interact.py``. From arigo at codespeak.net Thu Sep 6 18:06:55 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 6 Sep 2007 18:06:55 +0200 (CEST) Subject: [pypy-svn] r46375 - pypy/dist/pypy/doc/config Message-ID: <20070906160655.5C66F80F3@code0.codespeak.net> Author: arigo Date: Thu Sep 6 18:06:54 2007 New Revision: 46375 Modified: pypy/dist/pypy/doc/config/translation.sandbox.txt Log: Typo. Modified: pypy/dist/pypy/doc/config/translation.sandbox.txt ============================================================================== --- pypy/dist/pypy/doc/config/translation.sandbox.txt (original) +++ pypy/dist/pypy/doc/config/translation.sandbox.txt Thu Sep 6 18:06:54 2007 @@ -10,6 +10,6 @@ handle these operation requests, in any way it likes, allowing full virtualization. -For examples of controller process see +For examples of controller processes, see ``pypy/translator/sandbox/interact.py`` and ``pypy/translator/sandbox/pypy_interact.py``. From antocuni at codespeak.net Thu Sep 6 20:17:56 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 Sep 2007 20:17:56 +0200 (CEST) Subject: [pypy-svn] r46376 - in pypy/dist/pypy/translator: cli/test jvm/test oosupport/test_template Message-ID: <20070906181756.1B21180DC@code0.codespeak.net> Author: antocuni Date: Thu Sep 6 20:17:54 2007 New Revision: 46376 Added: pypy/dist/pypy/translator/oosupport/test_template/constant.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/test/test_constant.py pypy/dist/pypy/translator/jvm/test/test_constant.py Log: move test_constants to oosupport; remove duplicated code Modified: pypy/dist/pypy/translator/cli/test/test_constant.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_constant.py (original) +++ pypy/dist/pypy/translator/cli/test/test_constant.py Thu Sep 6 20:17:54 2007 @@ -1,130 +1,5 @@ +from pypy.translator.oosupport.test_template.constant import BaseTestConstant from pypy.translator.cli.test.runtest import CliTest -# used in tests below -class A: +class TestConstant(BaseTestConstant, CliTest): pass - - -class TestConstant(CliTest): - def test_char(self): - const = 'a' - def fn(): - return const - assert self.interpret(fn, []) == 'a' - - def test_void(self): - def fn(): - pass - assert self.interpret(fn, []) is None - - def test_tuple(self): - const = 1, 2 - def fn(): - return const - res = self.interpret(fn, []) - assert res.item0 == 1 - assert res.item1 == 2 - - def test_list(self): - const = [1, 2] - def fn(): - return const - res = self.ll_to_list(self.interpret(fn, [])) - assert res == [1, 2] - - def test_compound_const(self): - const = ([1, 2], [3, 4]) - def fn(): - return const - res = self.interpret(fn, []) - assert self.ll_to_list(res.item0) == [1, 2] - assert self.ll_to_list(res.item1) == [3, 4] - - def test_instance(self): - const = A() - def fn(): - return const - res = self.interpret(fn, []) - assert self.class_name(res) == 'A' - - def test_list_of_zeroes(self): - const = [0] * 10 - def fn(): - return const - res = self.ll_to_list(self.interpret(fn, [])) - assert res == const - - def test_list_of_instances(self): - const = [A()] - def fn(): - return const - res = self.ll_to_list(self.interpret(fn, [])) - assert self.class_name(res[0]) == 'A' - - def test_mix_string_and_char(self): - def fn(x): - if x < 0: - return 'a' - else: - return 'aa' - assert self.ll_to_string(self.interpret(fn, [-1])) == 'a' - assert self.ll_to_string(self.interpret(fn, [0])) == 'aa' - - def test_string_literal(self): - def fn(): - return 'hello "world"' - assert self.ll_to_string(self.interpret(fn, [])) == 'hello "world"' - - def test_string_literal2(self): - literals = ['\001\002\003', '\004\005\006'] - def fn(i): - s = literals[i] - return len(s), ord(s[0]) + ord(s[1]) + ord(s[2]) - res = self.interpret(fn, [0]) - assert res.item0 == 3 - assert res.item1 == 6 - res = self.interpret(fn, [1]) - assert res.item0 == 3 - assert res.item1 == 15 - - def test_float_special(self): - self._skip_win('inf & nan') - self._skip_powerpc('Suspected endian issue with '+ - 'representation of INF and NAN') - c = [float("inf"), float("nan")] - def fn(i): - return c[i]*2 == c[i] - def fn2(i): - return c[i] != c[i] - assert self.interpret(fn, [0]) == True - assert self.interpret(fn2, [1]) == True - - def test_customdict_circular(self): - from pypy.rlib.objectmodel import r_dict - def key_eq(a, b): - return a.x[0] == b.x[0] - def key_hash(a): - return ord(a.x[0]) - - class A: - def __init__(self, x): - self.x = x - a = A('foo') - a.dict = r_dict(key_eq, key_hash) - a.dict[a] = 42 - def fn(b): - if b: - s = A('foo') - else: - s = A('bar') - return a.dict[s] - assert self.interpret(fn, [True]) == 42 - - def test_multiple_step(self): - from pypy.translator.oosupport import constant - constant.MAX_CONST_PER_STEP = 2 - c1 = [1] - c2 = [2] - def fn(x, y): - return c1[x] + c2[y] - assert self.interpret(fn, [0, 0]) == 3 Modified: pypy/dist/pypy/translator/jvm/test/test_constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_constant.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_constant.py Thu Sep 6 20:17:54 2007 @@ -1,142 +1,8 @@ import py +from pypy.translator.oosupport.test_template.constant import BaseTestConstant from pypy.translator.jvm.test.runtest import JvmTest -# used in tests below -class A: - pass - - -class TestConstant(JvmTest): - def test_char(self): - const = 'a' - def fn(): - return const - assert self.interpret(fn, []) == 'a' - - def test_void(self): - def fn(): - pass - assert self.interpret(fn, []) is None - - def test_tuple(self): - const = 1, 2 - def fn(): - return const - res = self.interpret(fn, []) - assert res.item0 == 1 - assert res.item1 == 2 - - def test_list(self): - const = [1, 2] - def fn(): - return const - res = self.ll_to_list(self.interpret(fn, [])) - assert res == [1, 2] - - def test_compound_const(self): - const = ([1, 2], [3, 4]) - def fn(): - return const - res = self.interpret(fn, []) - assert self.ll_to_list(res.item0) == [1, 2] - assert self.ll_to_list(res.item1) == [3, 4] - - def test_instance(self): - const = A() - def fn(): - return const - res = self.interpret(fn, []) - assert self.class_name(res) == 'A' - - def test_list_of_zeroes(self): - const = [0] * 10 - def fn(): - return const - res = self.ll_to_list(self.interpret(fn, [])) - assert res == const - - def test_list_of_instances(self): - const = [A()] - def fn(): - return const - res = self.ll_to_list(self.interpret(fn, [])) - assert self.class_name(res[0]) == 'A' - - def test_mix_string_and_char(self): - def fn(x): - if x < 0: - return 'a' - else: - return 'aa' - assert self.ll_to_string(self.interpret(fn, [-1])) == 'a' - assert self.ll_to_string(self.interpret(fn, [0])) == 'aa' - - def test_string_literal(self): - def fn(): - return 'hello "world"' - assert self.ll_to_string(self.interpret(fn, [])) == 'hello "world"' - - def test_string_literal2(self): - literals = ['\001\002\003', '\004\005\006'] - def fn(i): - s = literals[i] - return len(s), ord(s[0]) + ord(s[1]) + ord(s[2]) - res = self.interpret(fn, [0]) - assert res.item0 == 3 - assert res.item1 == 6 - res = self.interpret(fn, [1]) - assert res.item0 == 3 - assert res.item1 == 15 - - def test_float_special(self): - self._skip_win('inf & nan') - self._skip_powerpc('Suspected endian issue with '+ - 'representation of INF and NAN') - c = [float("inf"), float("nan")] - def fn(i): - return c[i]*2 == c[i] - def fn2(i): - return c[i] != c[i] - assert self.interpret(fn, [0]) == True - assert self.interpret(fn2, [1]) == True - - def test_customdict_circular(self): - from pypy.rlib.objectmodel import r_dict - def key_eq(a, b): - return a.x[0] == b.x[0] - def key_hash(a): - return ord(a.x[0]) - - class A: - def __init__(self, x): - self.x = x - a = A('foo') - a.dict = r_dict(key_eq, key_hash) - a.dict[a] = 42 - def fn(b): - if b: - s = A('foo') - else: - s = A('bar') - return a.dict[s] - assert self.interpret(fn, [True]) == 42 - - def test_multiple_step(self): - from pypy.translator.oosupport import constant - constant.MAX_CONST_PER_STEP = 2 - c1 = [1] - c2 = [2] - def fn(x, y): - return c1[x] + c2[y] - assert self.interpret(fn, [0, 0]) == 3 +class TestConstant(BaseTestConstant, JvmTest): def test_many_constants(self): py.test.skip('Initializing a large constant list generates a function too large for the JVM limits') - N = 7500 - class A: - pass - mylist = [A() for i in range(N)] - def fn(x): - return mylist[x] - res = self.interpret(fn, [0]) - assert self.class_name(res) == 'A' Added: pypy/dist/pypy/translator/oosupport/test_template/constant.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/test_template/constant.py Thu Sep 6 20:17:54 2007 @@ -0,0 +1,137 @@ +# used in tests below +class A: + pass + +class BaseTestConstant: + def test_char(self): + const = 'a' + def fn(): + return const + assert self.interpret(fn, []) == 'a' + + def test_void(self): + def fn(): + pass + assert self.interpret(fn, []) is None + + def test_tuple(self): + const = 1, 2 + def fn(): + return const + res = self.interpret(fn, []) + assert res.item0 == 1 + assert res.item1 == 2 + + def test_list(self): + const = [1, 2] + def fn(): + return const + res = self.ll_to_list(self.interpret(fn, [])) + assert res == [1, 2] + + def test_compound_const(self): + const = ([1, 2], [3, 4]) + def fn(): + return const + res = self.interpret(fn, []) + assert self.ll_to_list(res.item0) == [1, 2] + assert self.ll_to_list(res.item1) == [3, 4] + + def test_instance(self): + const = A() + def fn(): + return const + res = self.interpret(fn, []) + assert self.class_name(res) == 'A' + + def test_list_of_zeroes(self): + const = [0] * 10 + def fn(): + return const + res = self.ll_to_list(self.interpret(fn, [])) + assert res == const + + def test_list_of_instances(self): + const = [A()] + def fn(): + return const + res = self.ll_to_list(self.interpret(fn, [])) + assert self.class_name(res[0]) == 'A' + + def test_mix_string_and_char(self): + def fn(x): + if x < 0: + return 'a' + else: + return 'aa' + assert self.ll_to_string(self.interpret(fn, [-1])) == 'a' + assert self.ll_to_string(self.interpret(fn, [0])) == 'aa' + + def test_string_literal(self): + def fn(): + return 'hello "world"' + assert self.ll_to_string(self.interpret(fn, [])) == 'hello "world"' + + def test_string_literal2(self): + literals = ['\001\002\003', '\004\005\006'] + def fn(i): + s = literals[i] + return len(s), ord(s[0]) + ord(s[1]) + ord(s[2]) + res = self.interpret(fn, [0]) + assert res.item0 == 3 + assert res.item1 == 6 + res = self.interpret(fn, [1]) + assert res.item0 == 3 + assert res.item1 == 15 + + def test_float_special(self): + self._skip_win('inf & nan') + self._skip_powerpc('Suspected endian issue with '+ + 'representation of INF and NAN') + c = [float("inf"), float("nan")] + def fn(i): + return c[i]*2 == c[i] + def fn2(i): + return c[i] != c[i] + assert self.interpret(fn, [0]) == True + assert self.interpret(fn2, [1]) == True + + def test_customdict_circular(self): + from pypy.rlib.objectmodel import r_dict + def key_eq(a, b): + return a.x[0] == b.x[0] + def key_hash(a): + return ord(a.x[0]) + + class A: + def __init__(self, x): + self.x = x + a = A('foo') + a.dict = r_dict(key_eq, key_hash) + a.dict[a] = 42 + def fn(b): + if b: + s = A('foo') + else: + s = A('bar') + return a.dict[s] + assert self.interpret(fn, [True]) == 42 + + def test_multiple_step(self): + from pypy.translator.oosupport import constant + constant.MAX_CONST_PER_STEP = 2 + c1 = [1] + c2 = [2] + def fn(x, y): + return c1[x] + c2[y] + assert self.interpret(fn, [0, 0]) == 3 + + def test_many_constants(self): + N = 7500 + class A: + pass + mylist = [A() for i in range(N)] + def fn(x): + return mylist[x] + res = self.interpret(fn, [0]) + assert self.class_name(res) == 'A' From simonb at codespeak.net Fri Sep 7 01:18:51 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Fri, 7 Sep 2007 01:18:51 +0200 (CEST) Subject: [pypy-svn] r46379 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070906231851.29C1680F6@code0.codespeak.net> Author: simonb Date: Fri Sep 7 01:18:49 2007 New Revision: 46379 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: broadcasting and slicing, many more cases work Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Fri Sep 7 01:18:49 2007 @@ -99,11 +99,10 @@ if len(s_index.items)>s_array.ndim: raise AnnotatorError("invalid index") if isinstance(s_value, SomeArray): - if s_value.ndim != ndim: - # XX allow broadcasting.. + if s_value.ndim > ndim: raise AnnotatorError("shape mismatch") - elif ndim > 0: - raise AnnotatorError("need to set from array") + #elif ndim > 0: + # raise AnnotatorError("need to set from array") def getitem((s_array, s_index)): ndim = pair(s_array, s_index).get_leftover_dim() Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Fri Sep 7 01:18:49 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.rmodel import Repr, inputconst +from pypy.rpython.rmodel import Repr, FloatRepr, inputconst from pypy.rpython.rrange import AbstractRangeRepr from pypy.rpython.rint import IntegerRepr from pypy.rpython.rlist import AbstractBaseListRepr @@ -70,7 +70,7 @@ ("index", NPY_INTP), ("size", NPY_INTP), ("coordinates", INDEXARRAY), - ("dims_m1", INDEXARRAY), + ("dims_m1", INDEXARRAY), # array of dimensions - 1 ("strides", INDEXARRAY), ("backstrides", INDEXARRAY), #("factors", INDEXARRAY), @@ -115,21 +115,22 @@ return it ll_iter_new._always_inline_ = True - def ll_iter_broadcast_to_shape(ITER, ao, shape, iter_reset=ll_iter_reset): - if ao.ndim > ndim: - raise Exception("array is not broadcastable to correct shape") # XX raise here ? - diff = j = ndim - ao.ndim + def ll_iter_broadcast_to_shape(ITER, ao, target_ao, iter_reset=ll_iter_reset): + "iterate over but broadcast to the shape of " + assert target_ao.ndim == ndim + delta = j = ndim - ao.ndim + shape = target_ao.shape for i in range(ao.ndim): if ao.shape[i] != 1 and ao.shape[i] != shape[j]: - raise Exception("array is not broadcastable to correct shape") # XX raise here ? + raise Exception("array is not broadcastable to correct shape") j += 1 it = malloc(ITER) - it.size = ll_mul_list(ao.shape, ndim) + it.size = ll_mul_list(target_ao.shape, ndim) it.nd_m1 = ndim - 1 - #it.factors[nd-1] = 1 + #it.factors[ndim-1] = 1 for i in unroll_ndim: - it.dims_m1[i] = ao.shape[i]-1 - k = i - diff + it.dims_m1[i] = shape[i]-1 + k = i - delta if k<0 or ao.shape[k] != shape[i]: #it.contiguous = False it.strides[i] = 0 @@ -137,7 +138,7 @@ it.strides[i] = ao.strides[k] it.backstrides[i] = it.strides[i] * it.dims_m1[i] #if i > 0: - #it.factors[nd-i-1] = it.factors[nd-i]*shape[nd-i] + #it.factors[ndim-i-1] = it.factors[nd-i]*shape[ndim-i] iter_reset(it, ao.dataptr) return it ll_iter_broadcast_to_shape._always_inline_ = True @@ -153,8 +154,7 @@ it.dataptr = direct_ptradd(it.dataptr, -it.backstrides[i]) ll_iter_next._always_inline_ = True -# return ll_iter_new, ll_iter_broadcast_to_shape, ll_iter_next - return ll_iter_new, ll_iter_next + return ll_iter_new, ll_iter_reset, ll_iter_broadcast_to_shape, ll_iter_next def ll_unary_op(p0, p1, op=lambda x:x): p0[0] = op(p1[0]) @@ -162,16 +162,19 @@ def ll_binary_op(p0, p1, p2, op=lambda x,y:x+y): p0[0] = op(p1[0], p2[0]) -def ll_array_unary_op(iter_new, iter_next, ITER, array0, array1): - assert array0.ndim == array1.ndim - it0 = iter_new(ITER, array0) - it1 = iter_new(ITER, array1) + +def ll_array_set(it0, it1, iter_next): assert it0.size == it1.size while it0.index < it0.size: it0.dataptr[0] = it1.dataptr[0] iter_next(it0) iter_next(it1) +def ll_array_set1(value, it, iter_next): + while it.index < it.size: + it.dataptr[0] = value + iter_next(it) + def dim_of_ITER(ITER): return ITER.TO.coordinates.length @@ -275,32 +278,101 @@ class __extend__(pairtype(ArrayRepr, ArrayRepr)): - def rtype_add((r_arr1, r_arr2), hop): - v_arr1, v_arr2 = hop.inputargs(r_arr1, r_arr2) + def rtype_add((r_array1, r_array2), hop): + v_arr1, v_arr2 = hop.inputargs(r_array1, r_array2) cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) return hop.gendirectcall(ll_add, cARRAY, v_arr1, v_arr2) #class __extend__(pairtype(ArrayRepr, Repr)): # <------ USE THIS ?? class __extend__(pairtype(ArrayRepr, IntegerRepr)): - def rtype_setitem((r_arr, r_int), hop): - v_array, v_index, v_item = hop.inputargs(r_arr, Signed, r_arr.item_repr) + def rtype_setitem((r_array, r_int), hop): + assert r_array.ndim == 1, "NotImplemented" + v_array, v_index, v_item = hop.inputargs(r_array, Signed, r_array.item_repr) return hop.gendirectcall(ll_setitem1, v_array, v_index, v_item) - def rtype_getitem((r_arr, r_int), hop): - v_array, v_index = hop.inputargs(r_arr, Signed) + def rtype_getitem((r_array, r_int), hop): + assert r_array.ndim == 1, "NotImplemented" + v_array, v_index = hop.inputargs(r_array, Signed) return hop.gendirectcall(ll_getitem1, v_array, v_index) +def gen_get_view_slc(r_array, r_slc, hop): # XX method on the pair type ? + ndim = r_array.ndim + rslice = hop.rtyper.type_system.rslice + def ll_get_view_slc(ARRAY, ao, slc): + array = ll_allocate(ARRAY, ndim) + dataptr = direct_arrayitems(ao.data) + src_i = 0 + tgt_i = 0 + if r_slc == rslice.startonly_slice_repr: + start = slc + size = ao.shape[src_i] + if start > size: + start = size + size -= start + dataptr = direct_ptradd(dataptr, start*ao.strides[src_i]) + array.shape[tgt_i] = size + array.strides[tgt_i] = ao.strides[src_i] + tgt_i += 1 + elif r_slc == rslice.startstop_slice_repr: + start = slc.start + stop = slc.stop + size = ao.shape[src_i] + if start > size: + start = size + dataptr = direct_ptradd(dataptr, start*ao.strides[src_i]) + if stop < size: + size = stop + size -= start + if size < 0: + size = 0 + array.shape[tgt_i] = size + array.strides[tgt_i] = ao.strides[src_i] + tgt_i += 1 + else: + assert 0 + src_i += 1 + # consume the rest of ndim as if we found more slices + while tgt_i < ndim: + array.shape[tgt_i] = ao.shape[src_i] + array.strides[tgt_i] = ao.strides[src_i] + tgt_i += 1 + src_i += 1 + assert tgt_i == ndim + array.dataptr = dataptr + array.data = ao.data # keep a ref + return array + return ll_get_view_slc + class __extend__(pairtype(ArrayRepr, AbstractSliceRepr)): - def rtype_setitem((r_arr, r_slc), hop): + def rtype_setitem((r_array, r_slc), hop): r_item = hop.args_r[2] - v_array, v_slc, v_item = hop.inputargs(r_arr, r_slc, r_item) - cITER = hop.inputconst(Void, r_arr.ITER.TO) - iter_new, iter_next = gen_iter_funcs(r_arr.ndim) - cnew = hop.inputconst(Void, iter_new) + v_array, v_slc, v_item = hop.inputargs(r_array, r_slc, r_item) + cITER = hop.inputconst(Void, r_array.ITER.TO) + cARRAY = hop.inputconst(Void, r_array.ARRAY.TO) + iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(r_array.ndim) cnext = hop.inputconst(Void, iter_next) - assert r_arr.ndim == r_item.ndim - return hop.gendirectcall(ll_array_unary_op, cnew, cnext, cITER, v_array, v_item) + creset = hop.inputconst(Void, iter_reset) +## Blech... it would be nice to reuse gen_get_view +## r_tuple = TupleRepr(hop.rtyper, [r_item]) # XX how to get this from rtyper ? +## get_view = gen_get_view(r_array, r_tuple, hop) +## # make a v_tuple here... +## v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_tuple) + get_view = gen_get_view_slc(r_array, r_slc, hop) + v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_slc) + v_it0 = hop.gendirectcall(iter_new, cITER, v_view, creset) + if isinstance(r_item, ArrayRepr): + if r_array.ndim == r_item.ndim: + v_it1 = hop.gendirectcall(iter_new, cITER, v_item, creset) + else: + v_it1 = hop.gendirectcall(iter_broadcast, cITER, v_item, v_array, creset) + assert r_array.ndim >= r_item.ndim + return hop.gendirectcall(ll_array_set, v_it0, v_it1, cnext) + elif isinstance(r_item, FloatRepr): + # setitem from scalar + return hop.gendirectcall(ll_array_set1, v_item, v_it0, cnext) + else: + raise TypeError("can't setitem from %s"%r_item) def gen_getset_item(ndim): unrolling_dims = unrolling_iterable(range(ndim)) @@ -320,26 +392,57 @@ return ll_get_item, ll_set_item -def get_view_ndim(r_tpl): - return len([r_item for r_item in r_tpl.items_r if isinstance(r_item, AbstractSliceRepr)]) - -def gen_get_view(r_tpl): - ndim = get_view_ndim(r_tpl) - unroll_r_tpl = unrolling_iterable(enumerate(r_tpl.items_r)) +def get_view_ndim(r_array, r_tuple): # XX method on the pair type ? + ndim = len([r_item for r_item in r_tuple.items_r if isinstance(r_item, AbstractSliceRepr)]) + ndim += r_array.ndim - len(r_tuple.items_r) + return ndim + +def gen_get_view(r_array, r_tuple, hop): # XX method on the pair type ? + ndim = get_view_ndim(r_array, r_tuple) + unroll_r_tuple = unrolling_iterable(enumerate(r_tuple.items_r)) + rslice = hop.rtyper.type_system.rslice def ll_get_view(ARRAY, ao, tpl): array = ll_allocate(ARRAY, ndim) dataptr = direct_arrayitems(ao.data) src_i = 0 tgt_i = 0 - for src_i, r_item in unroll_r_tpl: - if isinstance(r_item, IntegerRepr): - r_int = r_item + for src_i, r_key in unroll_r_tuple: + if isinstance(r_key, IntegerRepr): dataptr = direct_ptradd(dataptr, getattr(tpl, 'item%d'%src_i)*ao.strides[src_i]) - else: - r_slice = r_item - array.shape[tgt_i] = ao.shape[src_i] + elif r_key == rslice.startonly_slice_repr: + start = getattr(tpl, 'item%d'%src_i) + size = ao.shape[src_i] + if start > size: + start = size + size -= start + dataptr = direct_ptradd(dataptr, start*ao.strides[src_i]) + array.shape[tgt_i] = size + array.strides[tgt_i] = ao.strides[src_i] + tgt_i += 1 + elif r_key == rslice.startstop_slice_repr: + start = getattr(tpl, 'item%d'%src_i).start + stop = getattr(tpl, 'item%d'%src_i).stop + size = ao.shape[src_i] + if start > size: + start = size + dataptr = direct_ptradd(dataptr, start*ao.strides[src_i]) + if stop < size: + size = stop + size -= start + if size < 0: + size = 0 + array.shape[tgt_i] = size array.strides[tgt_i] = ao.strides[src_i] tgt_i += 1 + else: + assert 0 + src_i += 1 + # consume the rest of ndim as if we found more slices + while tgt_i < ndim: + array.shape[tgt_i] = ao.shape[src_i] + array.strides[tgt_i] = ao.strides[src_i] + tgt_i += 1 + src_i += 1 assert tgt_i == ndim array.dataptr = dataptr array.data = ao.data # keep a ref @@ -348,63 +451,70 @@ class __extend__(pairtype(ArrayRepr, AbstractTupleRepr)): - def rtype_getitem((r_arr, r_tpl), hop): - v_array, v_tuple = hop.inputargs(r_arr, r_tpl) - ndim = get_view_ndim(r_tpl) + def rtype_getitem((r_array, r_tpl), hop): + v_array, v_tuple = hop.inputargs(r_array, r_tpl) + ndim = get_view_ndim(r_array, r_tpl) if ndim == 0: # return a scalar - cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) - get_item, set_item = gen_getset_item(r_arr.ndim) + cARRAY = hop.inputconst(Void, r_array.ARRAY.TO) + get_item, set_item = gen_getset_item(r_array.ndim) return hop.gendirectcall(get_item, cARRAY, v_array, v_tuple) r_result = hop.r_result ARRAY = r_result.ARRAY assert dim_of_ARRAY(ARRAY) == ndim cARRAY = hop.inputconst(Void, ARRAY.TO) - ll_get_view = gen_get_view(r_tpl) + ll_get_view = gen_get_view(r_array, r_tpl, hop) return hop.gendirectcall(ll_get_view, cARRAY, v_array, v_tuple) - def rtype_setitem((r_arr, r_tpl), hop): + def rtype_setitem((r_array, r_tuple), hop): r_item = hop.args_r[2] - v_array, v_tuple, v_item = hop.inputargs(r_arr, r_tpl, r_item) - ndim = get_view_ndim(r_tpl) - if isinstance(r_item, ArrayRepr): - s_view = SomeArray(r_arr.s_array.typecode, ndim) + v_array, v_tuple, v_item = hop.inputargs(r_array, r_tuple, r_item) + ndim = get_view_ndim(r_array, r_tuple) + assert len(r_tuple.items_r) <= r_array.ndim + if ndim == 0: + # Set from scalar + assert isinstance(r_item, FloatRepr) + cARRAY = hop.inputconst(Void, r_array.ARRAY.TO) + get_item, set_item = gen_getset_item(r_array.ndim) + return hop.gendirectcall(set_item, cARRAY, v_array, v_tuple, v_item) + elif isinstance(r_item, ArrayRepr): + s_view = SomeArray(r_array.s_array.typecode, ndim) r_view = hop.rtyper.getrepr(s_view) cARRAY = hop.inputconst(Void, r_view.ARRAY.TO) - get_view = gen_get_view(r_tpl) + get_view = gen_get_view(r_array, r_tuple, hop) v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_tuple) - iter_new, iter_next = gen_iter_funcs(ndim) - assert ndim == r_item.ndim - assert dim_of_ITER(r_item.ITER) == dim_of_ITER(r_view.ITER) - cnew = hop.inputconst(Void, iter_new) + iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(ndim) + creset = hop.inputconst(Void, iter_reset) cnext = hop.inputconst(Void, iter_next) - cITER = hop.inputconst(Void, r_item.ITER.TO) - return hop.gendirectcall(ll_array_unary_op, cnew, cnext, cITER, v_view, v_item) + cITER = hop.inputconst(Void, r_view.ITER.TO) + v_it0 = hop.gendirectcall(iter_new, cITER, v_view, creset) + assert r_item.ndim <= ndim + if ndim == r_item.ndim: + v_it1 = hop.gendirectcall(iter_new, cITER, v_item, creset) + else: + v_it1 = hop.gendirectcall(iter_broadcast, cITER, v_item, v_view, creset) + return hop.gendirectcall(ll_array_set, v_it0, v_it1, cnext) else: - # Set from scalar - assert ndim == 0 - cARRAY = hop.inputconst(Void, r_arr.ARRAY.TO) - get_item, set_item = gen_getset_item(r_arr.ndim) - return hop.gendirectcall(set_item, cARRAY, v_array, v_tuple, v_item) + assert 0 class __extend__(pairtype(ArrayRepr, ArrayRepr)): - def convert_from_to((r_arr0, r_arr1), v, llops): + def convert_from_to((r_array0, r_array1), v, llops): assert 0 class __extend__(pairtype(AbstractBaseListRepr, ArrayRepr)): - def convert_from_to((r_lst, r_arr), v, llops): + def convert_from_to((r_lst, r_array), v, llops): if r_lst.listitem is None: return NotImplemented - if r_lst.item_repr != r_arr.item_repr: - assert 0, (r_lst, r_arr.item_repr) + if r_lst.item_repr != r_array.item_repr: + assert 0, (r_lst, r_array.item_repr) return NotImplemented - cARRAY = inputconst(lltype.Void, r_arr.lowleveltype.TO) + cARRAY = inputconst(lltype.Void, r_array.lowleveltype.TO) return llops.gendirectcall(ll_build_from_list, cARRAY, v) class __extend__(pairtype(AbstractRangeRepr, ArrayRepr)): - def convert_from_to((r_rng, r_arr), v, llops): - cARRAY = inputconst(lltype.Void, r_arr.lowleveltype.TO) + def convert_from_to((r_rng, r_array), v, llops): + cARRAY = inputconst(lltype.Void, r_array.lowleveltype.TO) return llops.gendirectcall(ll_build_from_list, cARRAY, v) def ll_allocate(ARRAY, ndim): Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Fri Sep 7 01:18:49 2007 @@ -187,6 +187,19 @@ assert type(s_array) == SomeArray assert s_array.ndim == 1 + def test_annotate_broadcast(self): + def f(): + a = numpy.empty((4,3), dtype='i') + b = numpy.array([33]) + a[:] = b + return a + t = TranslationContext() + a = t.buildannotator() + s_array = a.build_types(f, []) + assert type(s_array) == SomeArray + assert s_array.ndim == 2 + + from pypy.objspace.flow.model import checkgraph, flatten, Block, mkentrymap from pypy.translator.backendopt.malloc import LLTypeMallocRemover @@ -341,6 +354,66 @@ assert interpret(f, [0, 0]) == 0 assert interpret(f, [3, 4]) == 12 + def test_specialize_slice_1_0(self): + def f(): + a = numpy.zeros((12,), dtype='i') + a[:2] = 1 + a[5:9] = 2 + a[10:] = 3 + a[12:] = 99 + a[12:0] = 999 + return a + res = interpret(f, []) + data = [1,1,0,0,0,2,2,2,2,0,3,3] + for i in range(12): + assert res.dataptr[i] == data[i] + + def test_specialize_slice_1_1(self): + py.test.skip('this involves a runtime test to see if we need a broadcast iterator') + def f(): + a = numpy.zeros((6,), dtype='i') + a[:2] = numpy.array([1]) + a[5:9] = numpy.array([2]) + return a + res = interpret(f, []) + data = [1,1,0,0,0,2] + for i in range(6): + assert res.dataptr[i] == data[i] + + def test_specialize_slice_2_0(self): + py.test.skip('not implemented') + def f(): + a = numpy.zeros((12,), dtype='i').reshape((3,4)) + a[:2, 0] = 1 + return a + res = interpret(f, []) + data = [1,0,0,0,1,0,0,0,0,0,0,0] + for i in range(12): + assert res.dataptr[i] == data[i] + + def test_specialize_slice_2_1(self): + def f(): + a = numpy.zeros((12,), dtype='i').reshape((3,4)) + a[:2, 0] = numpy.array([1,2]) + a[1, 1:3] = numpy.array([4,5]) + a[0:1, 3:] = numpy.array([6,]) + return a + res = interpret(f, []) + data = [1, 0, 0, 6, 2, 4, 5, 0, 0, 0, 0, 0] + for i in range(12): + assert res.dataptr[i] == data[i] + + def test_specialize_slice_2_2(self): + def f(): + a = numpy.zeros((12,), dtype='i').reshape((3,4)) + b = numpy.array([1,2,3,4]).reshape((2,2)) + a[1:3, 2:] = b + return a + res = interpret(f, []) + data = [0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 3, 4] + for i in range(12): + assert res.dataptr[i] == data[i] + def test_specialize_view(self): def f(ii, jj): a = numpy.zeros((4, 5)) @@ -354,6 +427,45 @@ assert interpret(f, [2, 3]) == 2 + def test_specialize_view_implicit_slice(self): + def f(): + a = numpy.array(range(12)).reshape((3,4)) + b = a[0,] + return b + + res = interpret(f, []) + for i in range(4): + assert res.dataptr[i] == i + + def test_specialize_broadcast(self): + def f(): + a = numpy.empty((4,3), dtype='i') + b = numpy.array([33]) + a[:,:] = b + return a + res = interpret(f, []) + for i in range(4*3): + assert res.dataptr[i] == 33 + + def f(): + a = numpy.empty((4,3), dtype='i') + b = numpy.array([33]) + a[:,] = b + return a + res = interpret(f, []) + for i in range(4*3): + assert res.dataptr[i] == 33 + + def f(): + a = numpy.empty((4,3,2), dtype='i') + a[:] = numpy.array([33]) + a[0,:] = numpy.array([22]) + return a + res = interpret(f, []) + data = [22]*6 + [33]*18 + for i in range(3*4*2): + assert res.dataptr[i] == data[i] + def test_malloc_remove(self): py.test.skip('this test requires _always_inline_ magic hook') def f(): From tismer at codespeak.net Fri Sep 7 03:24:33 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Fri, 7 Sep 2007 03:24:33 +0200 (CEST) Subject: [pypy-svn] r46380 - in pypy/dist/pypy: interpreter lib lib/test2 module/_stackless Message-ID: <20070907012433.02A8280F9@code0.codespeak.net> Author: tismer Date: Fri Sep 7 03:24:31 2007 New Revision: 46380 Added: pypy/dist/pypy/lib/test2/pickledtasklet.py (contents, props changed) Modified: pypy/dist/pypy/interpreter/callmethod.py pypy/dist/pypy/lib/stackless.py pypy/dist/pypy/lib/test2/test_stackless.py pypy/dist/pypy/module/_stackless/coroutine.py Log: stackless pickling seems to be working, now. there is still a problem since we end up with a bus error. hopefully not mine... Modified: pypy/dist/pypy/interpreter/callmethod.py ============================================================================== --- pypy/dist/pypy/interpreter/callmethod.py (original) +++ pypy/dist/pypy/interpreter/callmethod.py Fri Sep 7 03:24:31 2007 @@ -14,6 +14,7 @@ from pypy.rlib.jit import we_are_jitted from pypy.interpreter.argument import Arguments from pypy.objspace.std import StdObjSpace +from pypy.rlib import rstack # for resume points def object_getattribute(space): @@ -66,6 +67,7 @@ n = nargs + (w_self is not None) try: w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) finally: f.dropvalues(nargs + 2) f.pushvalue(w_result) Modified: pypy/dist/pypy/lib/stackless.py ============================================================================== --- pypy/dist/pypy/lib/stackless.py (original) +++ pypy/dist/pypy/lib/stackless.py Fri Sep 7 03:24:31 2007 @@ -57,7 +57,7 @@ arguments *argl, **argd """ if self._frame is None or self._frame.dead: - self._frame = frame = GWrap() + self._frame = frame = MWrap(None)##GWrap() frame.coro = self if hasattr(self._frame, 'run') and self._frame.run: raise ValueError("cannot bind a bound coroutine") @@ -93,6 +93,9 @@ return _maincoro getcurrent = staticmethod(getcurrent) + def __reduce__(self): + raise TypeError, 'pickling is not possible based upon greenlets' + _maincoro = coroutine() maingreenlet = greenlet.getcurrent() _maincoro._frame = frame = MWrap(maingreenlet) @@ -169,6 +172,36 @@ def raise_(self): raise self.type, self.value, self.traceback +# +# helpers for pickling +# + +_stackless_primitive_registry = {} + +def register_stackless_primitive(thang, retval_expr='None'): + import types + func = thang + if isinstance(thang, types.MethodType): + func = thang.im_func + code = func.func_code + _stackless_primitive_registry[code] = retval_expr + # It is not too nice to attach info via the code object, but + # I can't think of a better solution without a real transform. + +def rewrite_stackless_primitive(coro_state, alive, tempval): + flags, state, thunk, parent = coro_state + for i, frame in enumerate(state): + retval_expr = _stackless_primitive_registry.get(frame.f_code) + if retval_expr: + # this tasklet needs to stop pickling here and return its value. + tempval = eval(retval_expr, globals(), frame.f_locals) + state = state[:i] + coro_state = flags, state, thunk, parent + return coro_state, alive, tempval + +# +# + class channel(object): """ A channel object is used for communication between tasklets. @@ -214,6 +247,7 @@ continue immediately, and the sender is put at the end of the runnables list. The above policy can be changed by setting channel flags. + XXX channel flags are not implemented, yet. """ receiver = getcurrent() willblock = not self.balance > 0 @@ -232,11 +266,14 @@ _scheduler_remove(getcurrent()) schedule() assert not receiver.blocked - + + # XXX wrong. This check should happen on every context switch, not here. msg = receiver.tempval if isinstance(msg, bomb): msg.raise_() return msg + + register_stackless_primitive(receive, retval_expr='receiver.tempval') def send_exception(self, exp_type, msg): self.send(bomb(exp_type, exp_type(msg))) @@ -273,6 +310,8 @@ schedule() assert not sender.blocked + register_stackless_primitive(send) + class tasklet(coroutine): """ A tasklet object represents a tiny task in a Python thread. @@ -282,7 +321,10 @@ """ tempval = None def __new__(cls, func=None, label=''): - return coroutine.__new__(cls) + res = coroutine.__new__(cls) + res.label = label + res._task_id = None + return res def __init__(self, func=None, label=''): coroutine.__init__(self) @@ -354,19 +396,42 @@ return self def run(self): - if _scheduler_contains(self): - return - else: - _scheduler_append(self) + self.insert() + _scheduler_switch(getcurrent(), self) + def insert(self): + if self.blocked: + raise RuntimeError, "You cannot run a blocked tasklet" + if not self.alive: + raise RuntimeError, "You cannot run an unbound(dead) tasklet" + _scheduler_append(self) + + def remove(self): + if self.blocked: + raise RuntimeError, "You cannot remove a blocked tasklet." + if self is getcurrent(): + raise RuntimeError, "The current tasklet cannot be removed." + # not sure if I will revive this " Use t=tasklet().capture()" + _scheduler_remove(self) + def __reduce__(self): - one, two, three = coroutine.__reduce__(self) + one, two, coro_state = coroutine.__reduce__(self) assert one is coroutine assert two == () - return tasklet, (), (three, self.alive, self.tempval) + # we want to get rid of the parent thing. + # for now, we just drop it + a, b, c, d = coro_state + if d: + assert isinstance(d, coroutine) + coro_state = a, b, c, None + coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval) + inst_dict = self.__dict__.copy() + del inst_dict['tempval'] + return self.__class__, (), (coro_state, alive, tempval, inst_dict) - def __setstate__(self, (coro_state, alive, tempval)): + def __setstate__(self, (coro_state, alive, tempval, inst_dict)): coroutine.__setstate__(self, coro_state) + self.__dict__.update(inst_dict) self.alive = alive self.tempval = tempval Added: pypy/dist/pypy/lib/test2/pickledtasklet.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/test2/pickledtasklet.py Fri Sep 7 03:24:31 2007 @@ -0,0 +1,27 @@ +import pickle, sys +import stackless + +ch = stackless.channel() + +def recurs(depth, level=1): + print 'enter level %s%d' % (level*' ', level) + if level >= depth: + ch.send('hi') + if level < depth: + recurs(depth, level+1) + print 'leave level %s%d' % (level*' ', level) + +def demo(depth): + t = stackless.tasklet(recurs)(depth) + print ch.receive() + pickle.dump(t, file('tasklet.pickle', 'wb')) + +if __name__ == '__main__': + if len(sys.argv) > 1: + t = pickle.load(file(sys.argv[1], 'rb')) + t.insert() + else: + t = stackless.tasklet(demo)(14) + stackless.run() + +# remark: think of fixing cells etc. on the sprint Modified: pypy/dist/pypy/lib/test2/test_stackless.py ============================================================================== --- pypy/dist/pypy/lib/test2/test_stackless.py (original) +++ pypy/dist/pypy/lib/test2/test_stackless.py Fri Sep 7 03:24:31 2007 @@ -7,25 +7,44 @@ cls.space = space def test_pickle(self): - import pickle, sys - import stackless - - ch = stackless.channel() - - def recurs(depth, level=1): - print 'enter level %s%d' % (level*' ', level) - if level >= depth: - ch.send('hi') - if level < depth: - recurs(depth, level+1) - print 'leave level %s%d' % (level*' ', level) - - def demo(depth): - t = stackless.tasklet(recurs)(depth) - print ch.receive() - blob = pickle.dumps(t) - - t = stackless.tasklet(demo)(14) - stackless.run() - -# remark: think of fixing cells etc. on the sprint + import new, sys + + mod = new.module('mod') + sys.modules['mod'] = mod + try: + exec ''' +import pickle, sys +import stackless +lev = 14 + +ch = stackless.channel() +seen = [] + +def recurs(depth, level=1): + print 'enter level %s%d' % (level*' ', level) + seen.append(level) + if level >= depth: + ch.send('hi') + if level < depth: + recurs(depth, level+1) + seen.append(level) + print 'leave level %s%d' % (level*' ', level) + +def demo(depth): + t = stackless.tasklet(recurs)(depth) + print ch.receive() + global blob + blob = pickle.dumps(t) + +t = stackless.tasklet(demo)(lev) +stackless.run() +assert seen == range(1, lev+1) + range(lev, 0, -1) +print "now running the clone" +tt = pickle.loads(blob) +tt.insert() +seen = [] +stackless.run() +assert seen == range(lev, 0, -1) +''' in mod.__dict__ + finally: + del sys.modules['mod'] Modified: pypy/dist/pypy/module/_stackless/coroutine.py ============================================================================== --- pypy/dist/pypy/module/_stackless/coroutine.py (original) +++ pypy/dist/pypy/module/_stackless/coroutine.py Fri Sep 7 03:24:31 2007 @@ -219,7 +219,10 @@ ec) instr = frame.last_instr opcode = ord(code[instr]) - assert opcode == pythonopcode.opmap['CALL_FUNCTION'] + map = pythonopcode.opmap + call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'], map['CALL_FUNCTION_VAR'], + map['CALL_FUNCTION_VAR_KW'], map['CALL_METHOD']] + assert opcode in call_ops # ("dispatch_call", self, co_code, next_instr, ec) chain = resume_state_create(chain, "dispatch_call", frame, code, instr+3, ec) @@ -229,8 +232,12 @@ # Only positional arguments nargs = oparg & 0xff # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) - chain = resume_state_create(chain, 'CALL_FUNCTION', frame, - nargs) + if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + else: + chain = resume_state_create(chain, 'CALL_FUNCTION', frame, + nargs) else: # case2: ("call_function", f, returns=w_result) chain = resume_state_create(chain, 'call_function', frame) From arigo at codespeak.net Fri Sep 7 09:58:33 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 09:58:33 +0200 (CEST) Subject: [pypy-svn] r46381 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907075833.A4C1D8100@code0.codespeak.net> Author: arigo Date: Fri Sep 7 09:58:32 2007 New Revision: 46381 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py Log: Enable by default sanboxed usage of the three time.*() functions of the simple 'time' module of PyPy. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 09:58:32 2007 @@ -5,7 +5,7 @@ """ import py -import marshal, sys, os, posixpath, errno, stat +import marshal, sys, os, posixpath, errno, stat, time from pypy.rpython.module.ll_os_stat import s_StatResult from pypy.tool.ansi_print import AnsiLog from pypy.rlib.rarithmetic import r_longlong @@ -231,6 +231,24 @@ return len(data) raise OSError("trying to write to fd %d" % (fd,)) + # let's allow access to the real time + def do_ll_time__ll_time_sleep(self, seconds): + time.sleep(seconds) + + def do_ll_time__ll_time_time(self): + return time.time() + + def do_ll_time__ll_time_clock(self): + # measuring the CPU time of the controller process has + # not much meaning, so let's emulate this and return + # the real time elapsed since the first call to clock() + # (this is one of the behaviors allowed by the docs) + try: + starttime = self.starttime + except AttributeError: + starttime = self.starttime = time.time() + return time.time() - starttime + class VirtualizedSandboxedProc(SandboxedProc): """Control a virtualized sandboxed process, which is given a custom From arigo at codespeak.net Fri Sep 7 10:42:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 10:42:34 +0200 (CEST) Subject: [pypy-svn] r46384 - in pypy/dist/pypy: rlib rpython rpython/lltypesystem translator/c translator/c/test translator/cli Message-ID: <20070907084234.0C405810F@code0.codespeak.net> Author: arigo Date: Fri Sep 7 10:42:34 2007 New Revision: 46384 Modified: pypy/dist/pypy/rlib/rgc.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/test/test_boehm.py pypy/dist/pypy/translator/cli/opcodes.py Log: Added rgc.set_max_heap_size(), turning into an operation that is so far only implemented by the Boehm GC policy (it is a no-op otherwise). Modified: pypy/dist/pypy/rlib/rgc.py ============================================================================== --- pypy/dist/pypy/rlib/rgc.py (original) +++ pypy/dist/pypy/rlib/rgc.py Fri Sep 7 10:42:34 2007 @@ -1,5 +1,16 @@ +import gc from pypy.rpython.extregistry import ExtRegistryEntry # ____________________________________________________________ +# General GC features + +collect = gc.collect + +def set_max_heap_size(nbytes): + """Limit the heap size to n bytes. + So far only implemented by the Boehm GC.""" + pass + +# ____________________________________________________________ # Framework GC features class GcPool(object): @@ -127,7 +138,6 @@ return rtuple.newtuple(hop.llops, r_tuple, [v_gcobject, v_pool]) # Support for collection. -import gc class CollectEntry(ExtRegistryEntry): _about_ = gc.collect @@ -139,4 +149,15 @@ def specialize_call(self, hop): return hop.genop('gc__collect', [], resulttype=hop.r_result) +class SetMaxHeapSizeEntry(ExtRegistryEntry): + _about_ = set_max_heap_size + def compute_result_annotation(self, s_nbytes): + from pypy.annotation import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + [v_nbytes] = hop.inputargs(lltype.Signed) + return hop.genop('gc_set_max_heap_size', [v_nbytes], + resulttype=lltype.Void) Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Sep 7 10:42:34 2007 @@ -741,6 +741,9 @@ def op_gc_reload_possibly_moved(self, newaddr, ptr): raise NotImplementedError("gc_reload_possibly_moved") + def op_gc_set_max_heap_size(self, maxsize): + raise NotImplementedError("gc_set_max_heap_size") + def op_yield_current_frame_to_caller(self): raise NotImplementedError("yield_current_frame_to_caller") Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Fri Sep 7 10:42:34 2007 @@ -384,6 +384,7 @@ 'gc_protect': LLOp(), 'gc_unprotect': LLOp(), 'gc_reload_possibly_moved': LLOp(), + 'gc_set_max_heap_size': LLOp(), # experimental operations in support of thread cloning, only # implemented by the Mark&Sweep GC 'gc_x_swap_pool': LLOp(canraise=(MemoryError,), canunwindgc=True), Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Fri Sep 7 10:42:34 2007 @@ -67,6 +67,9 @@ expr = funcgen.expr(op.args[0]) return 'Py_XDECREF(%s);' % expr + def OP_GC_SET_MAX_HEAP_SIZE(self, funcgen, op): + return '' + class RefcountingInfo: static_deallocator = None @@ -223,6 +226,10 @@ def OP_GC__COLLECT(self, funcgen, op): return 'GC_gcollect(); GC_invoke_finalizers();' + def OP_GC_SET_MAX_HEAP_SIZE(self, funcgen, op): + nbytes = funcgen.expr(op.args[0]) + return 'GC_set_max_heap_size(%s);' % (nbytes,) + class BoehmGcRuntimeTypeInfo_OpaqueNode(ContainerNode): nodekind = 'boehm rtti' globalcontainer = True Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Fri Sep 7 10:42:34 2007 @@ -266,6 +266,23 @@ res = c_var_size() assert res == 0 + def test_gc_set_max_heap_size(self): + def g(n): + return 'x' * n + def fn(): + from pypy.rlib import rgc + rgc.set_max_heap_size(500000) + s1 = s2 = s3 = None + try: + s1 = g(10000) + s2 = g(100000) + s3 = g(1000000) + except MemoryError: + pass + return (s1 is not None) + (s2 is not None) + (s3 is not None) + c_fn = self.getcompiled(fn, []) + res = c_fn() + assert res == 2 class TestUsingExactBoehm(TestUsingBoehm): Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Fri Sep 7 10:42:34 2007 @@ -62,6 +62,7 @@ 'cast_ptr_to_weakadr': [PushAllArgs, 'newobj instance void class %s::.ctor(object)' % WEAKREF], 'cast_weakadr_to_ptr': [CastWeakAdrToPtr], 'gc__collect': 'call void class [mscorlib]System.GC::Collect()', + 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, 'debug_assert': Ignore, From arigo at codespeak.net Fri Sep 7 10:52:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 10:52:59 +0200 (CEST) Subject: [pypy-svn] r46385 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070907085259.98AEF810F@code0.codespeak.net> Author: arigo Date: Fri Sep 7 10:52:57 2007 New Revision: 46385 Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py (props changed) pypy/dist/pypy/translator/jvm/test/test_constant.py (props changed) pypy/dist/pypy/translator/jvm/test/test_exception.py (props changed) pypy/dist/pypy/translator/jvm/test/test_float.py (props changed) pypy/dist/pypy/translator/jvm/test/test_int.py (props changed) pypy/dist/pypy/translator/jvm/test/test_oo.py (props changed) pypy/dist/pypy/translator/jvm/test/test_op.py (props changed) pypy/dist/pypy/translator/jvm/test/test_overflow.py (props changed) pypy/dist/pypy/translator/jvm/test/test_primitive.py (props changed) pypy/dist/pypy/translator/jvm/test/test_range.py (props changed) pypy/dist/pypy/translator/jvm/test/test_runtest.py (props changed) pypy/dist/pypy/translator/jvm/test/test_streamio.py (props changed) pypy/dist/pypy/translator/jvm/test/test_tuple.py (props changed) Log: fixeol From arigo at codespeak.net Fri Sep 7 10:53:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 10:53:29 +0200 (CEST) Subject: [pypy-svn] r46386 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070907085329.F28B08110@code0.codespeak.net> Author: arigo Date: Fri Sep 7 10:53:29 2007 New Revision: 46386 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ (props changed) Log: Ignore the *.class files. From arigo at codespeak.net Fri Sep 7 11:53:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 11:53:46 +0200 (CEST) Subject: [pypy-svn] r46387 - in pypy/dist/pypy/tool: . test Message-ID: <20070907095346.021988100@code0.codespeak.net> Author: arigo Date: Fri Sep 7 11:53:45 2007 New Revision: 46387 Added: pypy/dist/pypy/tool/killsubprocess.py (contents, props changed) pypy/dist/pypy/tool/test/test_killsubprocess.py (contents, props changed) Log: A way to kill subprocess on Unix and Windows, from Python cookbook entries. I think this method is reliable against race conditions like the subprocess dying just at the wrong moment and another process reusing the pid. Added: pypy/dist/pypy/tool/killsubprocess.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/killsubprocess.py Fri Sep 7 11:53:45 2007 @@ -0,0 +1,24 @@ +"""Kill a subprocess created by subprocess.Popen(). +The two Windows versions come from the Python cookbook. +""" +import sys, os + +if sys.platform != "win32": + import signal + assert hasattr(os, 'kill') + + def killsubprocess(process): + if process.poll() is None: + os.kill(process.pid, signal.SIGTERM) + +else: + # on Windows, we need either win32api or ctypes + try: + import ctypes + TerminateProcess = ctypes.windll.kernel32.TerminateProcess + except ImportError: + from win32api import TerminateProcess + + def killsubprocess(process): + if process.poll() is None: + TerminateProcess(int(process._handle), -1) Added: pypy/dist/pypy/tool/test/test_killsubprocess.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/test/test_killsubprocess.py Fri Sep 7 11:53:45 2007 @@ -0,0 +1,34 @@ +import sys, time +from py.compat import subprocess +from pypy.tool.killsubprocess import killsubprocess + +def waitdead(process): + for i in range(50): + time.sleep(0.1) + if process.poll() is not None: + break # ok + else: + raise AssertionError("the subprocess did not die within 5 seconds") + +def test_killsubprocess(): + popen = subprocess.Popen([sys.executable, '-c', 'raw_input()'], + stdin=subprocess.PIPE) + time.sleep(0.9) + assert popen.poll() is None + assert popen.poll() is None + killsubprocess(popen) + waitdead(popen) + +def test_already_dead_but_no_poll(): + popen = subprocess.Popen([sys.executable, '-c', 'pass'], + stdin=subprocess.PIPE) + time.sleep(3) # a safe margin to be sure the subprocess is already dead + killsubprocess(popen) + assert popen.poll() is not None + +def test_already_dead_and_polled(): + popen = subprocess.Popen([sys.executable, '-c', 'pass'], + stdin=subprocess.PIPE) + waitdead(popen) + killsubprocess(popen) + assert popen.poll() is not None From arigo at codespeak.net Fri Sep 7 11:58:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 11:58:52 +0200 (CEST) Subject: [pypy-svn] r46388 - in pypy/dist/pypy/translator: goal sandbox Message-ID: <20070907095852.3D1188100@code0.codespeak.net> Author: arigo Date: Fri Sep 7 11:58:51 2007 New Revision: 46388 Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py pypy/dist/pypy/translator/sandbox/pypy_interact.py Log: Add a --heapsize option to pypy_interact that uses a kind-of-hidden --heapsize option of the sandboxed subprocess. Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Fri Sep 7 11:58:51 2007 @@ -31,6 +31,13 @@ #debug("entry point starting") #for arg in argv: # debug(" argv -> " + arg) + if len(argv) > 2 and argv[1] == '--heapsize': + # Undocumented option, handled at interp-level. + # It has silently no effect in non-Boehm translations. + # At the moment this option exists mainly to support sandboxing. + from pypy.rlib import rgc + rgc.set_max_heap_size(int(argv[2])) + argv = argv[:1] + argv[3:] try: try: space.call_function(w_run_toplevel, w_call_startup_gateway) Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Fri Sep 7 11:58:51 2007 @@ -8,6 +8,9 @@ Options: --tmp=DIR the real directory that corresponds to the virtual /tmp, which is the virtual current dir (always read-only for now) + --heapsize=N limit memory usage to N bytes, or kilo- mega- giga-bytes + with the 'k', 'm' or 'g' suffix respectively. + ATM this only works if the sandboxed executable uses Boehm. """ import sys, os @@ -53,8 +56,10 @@ if __name__ == '__main__': from getopt import getopt # and not gnu_getopt! - options, arguments = getopt(sys.argv[1:], 't:h', ['tmp=', 'help']) + options, arguments = getopt(sys.argv[1:], 't:h', + ['tmp=', 'heapsize=', 'help']) tmpdir = None + extraoptions = [] def help(): print >> sys.stderr, __doc__ @@ -66,6 +71,21 @@ if not os.path.isdir(value): raise OSError("%r is not a directory" % (value,)) tmpdir = value + elif option == '--heapsize': + value = value.lower() + if value.endswith('k'): + bytes = int(value[:-1]) * 1024 + elif value.endswith('m'): + bytes = int(value[:-1]) * 1024 * 1024 + elif value.endswith('g'): + bytes = int(value[:-1]) * 1024 * 1024 * 1024 + else: + bytes = int(value) + if bytes <= 0: + raise ValueError + if bytes > sys.maxint: + raise OverflowError("--heapsize maximum is %d" % sys.maxint) + extraoptions[:0] = ['--heapsize', str(bytes)] elif option in ['-h', '--help']: help() else: @@ -74,5 +94,6 @@ if len(arguments) < 1: help() - sandproc = PyPySandboxedProc(arguments[0], arguments[1:], tmpdir=tmpdir) + sandproc = PyPySandboxedProc(arguments[0], extraoptions + arguments[1:], + tmpdir=tmpdir) sandproc.interact() From arigo at codespeak.net Fri Sep 7 12:04:33 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 12:04:33 +0200 (CEST) Subject: [pypy-svn] r46389 - in pypy/dist/pypy/translator/sandbox: . test Message-ID: <20070907100433.2DB648119@code0.codespeak.net> Author: arigo Date: Fri Sep 7 12:04:32 2007 New Revision: 46389 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Log: Move the flush() out of write_message() because we often write two messages at once. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 12:04:32 2007 @@ -45,7 +45,6 @@ buf = [] get_marshaller(resulttype)(buf, msg) g.write(''.join(buf)) - g.flush() # keep the table in sync with rsandbox.reraise_error() EXCEPTION_TABLE = [ @@ -69,6 +68,7 @@ if error is None: error = errno.EPERM write_message(g, error) + g.flush() break else: # just re-raise the exception @@ -154,6 +154,7 @@ log.result(shortrepr(answer)) write_message(self.popen.stdin, 0) # error code - 0 for ok write_message(self.popen.stdin, answer, resulttype) + self.popen.stdin.flush() returncode = self.popen.wait() return returncode Modified: pypy/dist/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Fri Sep 7 12:04:32 2007 @@ -17,6 +17,7 @@ else: write_message(g, 0) write_message(g, result, resulttype) + g.flush() def test_open_dup(): From arigo at codespeak.net Fri Sep 7 13:28:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 13:28:05 +0200 (CEST) Subject: [pypy-svn] r46390 - pypy/dist/pypy/lib Message-ID: <20070907112805.72E798116@code0.codespeak.net> Author: arigo Date: Fri Sep 7 13:28:04 2007 New Revision: 46390 Modified: pypy/dist/pypy/lib/_marshal.py Log: Accept a 'version' argument in marshal.dump() and dumps(). It is ignored at the moment because we always dump in the format version 0 (but we can always load version 1 too). Modified: pypy/dist/pypy/lib/_marshal.py ============================================================================== --- pypy/dist/pypy/lib/_marshal.py (original) +++ pypy/dist/pypy/lib/_marshal.py Fri Sep 7 13:28:04 2007 @@ -663,7 +663,10 @@ # # user interface -def dump(x, f): +version = 1 + +def dump(x, f, version=version): + # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) @@ -671,7 +674,8 @@ um = _Unmarshaller(f.read) return um.load() -def dumps(x): +def dumps(x, version=version): + # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] m = _Marshaller(buffer.append) m.dump(x) From arigo at codespeak.net Fri Sep 7 13:34:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 13:34:07 +0200 (CEST) Subject: [pypy-svn] r46391 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907113407.550ED8105@code0.codespeak.net> Author: arigo Date: Fri Sep 7 13:34:07 2007 New Revision: 46391 Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py pypy/dist/pypy/translator/sandbox/sandlib.py Log: A --timeout option for pypy_interact.py. Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Fri Sep 7 13:34:07 2007 @@ -11,6 +11,7 @@ --heapsize=N limit memory usage to N bytes, or kilo- mega- giga-bytes with the 'k', 'm' or 'g' suffix respectively. ATM this only works if the sandboxed executable uses Boehm. + --timeout=N limit execution time to N (real-time) seconds. """ import sys, os @@ -57,8 +58,9 @@ if __name__ == '__main__': from getopt import getopt # and not gnu_getopt! options, arguments = getopt(sys.argv[1:], 't:h', - ['tmp=', 'heapsize=', 'help']) + ['tmp=', 'heapsize=', 'timeout=', 'help']) tmpdir = None + timeout = None extraoptions = [] def help(): @@ -86,6 +88,8 @@ if bytes > sys.maxint: raise OverflowError("--heapsize maximum is %d" % sys.maxint) extraoptions[:0] = ['--heapsize', str(bytes)] + elif option == '--timeout': + timeout = int(value) elif option in ['-h', '--help']: help() else: @@ -96,4 +100,6 @@ sandproc = PyPySandboxedProc(arguments[0], extraoptions + arguments[1:], tmpdir=tmpdir) + if timeout is not None: + sandproc.settimeout(timeout) sandproc.interact() Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 13:34:07 2007 @@ -5,7 +5,8 @@ """ import py -import marshal, sys, os, posixpath, errno, stat, time +import sys, os, posixpath, errno, stat, time +from pypy.lib import marshal # see below from pypy.rpython.module.ll_os_stat import s_StatResult from pypy.tool.ansi_print import AnsiLog from pypy.rlib.rarithmetic import r_longlong @@ -23,6 +24,13 @@ py.log.setconsumer("sandlib", MyAnsiLog()) +# Note: we use pypy.lib.marshal instead of the built-in marshal +# for two reasons. The built-in module could be made to segfault +# or be attackable in other ways by sending malicious input to +# load(). Also, marshal.load(f) blocks with the GIL held when +# f is a pipe with no data immediately avaialble, preventing the +# _waiting_thread to run. + def read_message(f, timeout=None): # warning: 'timeout' is not really reliable and should only be used # for testing. Also, it doesn't work if the file f does any buffering. @@ -106,12 +114,62 @@ stdout=subprocess.PIPE, close_fds=True, env={}) + self.popenlock = None + self.currenttimeout = None + + def withlock(self, function, *args, **kwds): + lock = self.popenlock + if lock is not None: + lock.acquire() + try: + return function(*args, **kwds) + finally: + if lock is not None: + lock.release() + + def settimeout(self, timeout): + """Start a timeout that will kill the subprocess after the given + amount of time. Only one timeout can be active at a time. + """ + import thread + from pypy.tool.killsubprocess import killsubprocess + + def _waiting_thread(): + while True: + t = self.currenttimeout + if t is None: + return # cancelled + delay = t - time.time() + if delay <= 0.0: + break # expired! + time.sleep(min(delay*1.001, 1)) + self.withlock(killsubprocess, self.popen) + + def _settimeout(): + need_new_thread = self.currenttimeout is None + self.currenttimeout = time.time() + timeout + if need_new_thread: + thread.start_new_thread(_waiting_thread, ()) + + if self.popenlock is None: + self.popenlock = thread.allocate_lock() + self.withlock(_settimeout) + + def canceltimeout(self): + """Cancel the current timeout.""" + self.currenttimeout = None def poll(self): - return self.popen.poll() + returncode = self.withlock(self.popen.poll) + if returncode is not None: + self.canceltimeout() + return returncode def wait(self): - return self.popen.wait() + returncode = self.withlock(self.popen.wait) + if returncode is not None: + self.canceltimeout() + return returncode def handle_forever(self): returncode = self.handle_until_return() @@ -120,20 +178,22 @@ returncode,)) def handle_until_return(self): + child_stdin = self.popen.stdin + child_stdout = self.popen.stdout if self.os_level_sandboxing and sys.platform.startswith('linux2'): # rationale: we wait until the child process started completely, # letting the C library do any system calls it wants for # initialization. When the RPython code starts up, it quickly # does its first system call. At this point we turn seccomp on. import select - select.select([self.popen.stdout], [], []) + select.select([child_stdout], [], []) f = open('/proc/%d/seccomp' % self.popen.pid, 'w') print >> f, 1 f.close() while True: try: - fnname = read_message(self.popen.stdout) - args = read_message(self.popen.stdout) + fnname = read_message(child_stdout) + args = read_message(child_stdout) except EOFError, e: break if self.debug: @@ -143,7 +203,7 @@ answer, resulttype = self.handle_message(fnname, *args) except Exception, e: tb = sys.exc_info()[2] - write_exception(self.popen.stdin, e, tb) + write_exception(child_stdin, e, tb) if self.debug: if str(e): log.exception('%s: %s' % (e.__class__.__name__, e)) @@ -152,10 +212,10 @@ else: if self.debug: log.result(shortrepr(answer)) - write_message(self.popen.stdin, 0) # error code - 0 for ok - write_message(self.popen.stdin, answer, resulttype) - self.popen.stdin.flush() - returncode = self.popen.wait() + write_message(child_stdin, 0) # error code - 0 for ok + write_message(child_stdin, answer, resulttype) + child_stdin.flush() + returncode = self.wait() return returncode def handle_message(self, fnname, *args): From arigo at codespeak.net Fri Sep 7 13:43:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 13:43:47 +0200 (CEST) Subject: [pypy-svn] r46393 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907114347.BEE82810C@code0.codespeak.net> Author: arigo Date: Fri Sep 7 13:43:47 2007 New Revision: 46393 Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py pypy/dist/pypy/translator/sandbox/sandlib.py Log: Allow the timeout to actively stop a pypy_interact that is waiting for external input (e.g. the next interactive command). Without this, the sandboxed child is killed but the outer process only notices when we press Enter. Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Fri Sep 7 13:43:47 2007 @@ -101,5 +101,5 @@ sandproc = PyPySandboxedProc(arguments[0], extraoptions + arguments[1:], tmpdir=tmpdir) if timeout is not None: - sandproc.settimeout(timeout) + sandproc.settimeout(timeout, interrupt_main=True) sandproc.interact() Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 13:43:47 2007 @@ -127,7 +127,7 @@ if lock is not None: lock.release() - def settimeout(self, timeout): + def settimeout(self, timeout, interrupt_main=False): """Start a timeout that will kill the subprocess after the given amount of time. Only one timeout can be active at a time. """ @@ -144,6 +144,12 @@ break # expired! time.sleep(min(delay*1.001, 1)) self.withlock(killsubprocess, self.popen) + if interrupt_main: + if hasattr(os, 'kill'): + import signal + os.kill(os.getpid(), signal.SIGINT) + else: + thread.interrupt_main() def _settimeout(): need_new_thread = self.currenttimeout is None From arigo at codespeak.net Fri Sep 7 13:49:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 13:49:08 +0200 (CEST) Subject: [pypy-svn] r46394 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907114908.C17268100@code0.codespeak.net> Author: arigo Date: Fri Sep 7 13:49:07 2007 New Revision: 46394 Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py pypy/dist/pypy/translator/sandbox/sandlib.py Log: Double-check that the subprocess is killed when the controller ends. Protects against the subprocess sending broken data causing an exception in the controller, causing the timeout to be cancelled. The subprocess would then be able to consume CPU cycles forever (without any means of communication, though). Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Fri Sep 7 13:49:07 2007 @@ -102,4 +102,7 @@ tmpdir=tmpdir) if timeout is not None: sandproc.settimeout(timeout, interrupt_main=True) - sandproc.interact() + try: + sandproc.interact() + finally: + sandproc.kill() Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 13:49:07 2007 @@ -11,6 +11,7 @@ from pypy.tool.ansi_print import AnsiLog from pypy.rlib.rarithmetic import r_longlong from py.compat import subprocess +from pypy.tool.killsubprocess import killsubprocess class MyAnsiLog(AnsiLog): KW_TO_COLOR = { @@ -132,7 +133,6 @@ amount of time. Only one timeout can be active at a time. """ import thread - from pypy.tool.killsubprocess import killsubprocess def _waiting_thread(): while True: @@ -143,7 +143,7 @@ if delay <= 0.0: break # expired! time.sleep(min(delay*1.001, 1)) - self.withlock(killsubprocess, self.popen) + self.kill() if interrupt_main: if hasattr(os, 'kill'): import signal @@ -177,6 +177,9 @@ self.canceltimeout() return returncode + def kill(self): + self.withlock(killsubprocess, self.popen) + def handle_forever(self): returncode = self.handle_until_return() if returncode != 0: From arigo at codespeak.net Fri Sep 7 14:13:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 14:13:04 +0200 (CEST) Subject: [pypy-svn] r46395 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907121304.F2139810C@code0.codespeak.net> Author: arigo Date: Fri Sep 7 14:13:02 2007 New Revision: 46395 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py Log: Good suggestion by mwh: hide the spamming amounts of reads and writes to stdin and stdout in interactive sessions. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 14:13:02 2007 @@ -205,7 +205,7 @@ args = read_message(child_stdout) except EOFError, e: break - if self.debug: + if self.debug and not self.is_spam(fnname, *args): log.call('%s(%s)' % (fnname, ', '.join([shortrepr(x) for x in args]))) try: @@ -219,7 +219,7 @@ else: log.exception('%s' % (e.__class__.__name__,)) else: - if self.debug: + if self.debug and not self.is_spam(fnname, *args): log.result(shortrepr(answer)) write_message(child_stdin, 0) # error code - 0 for ok write_message(child_stdin, answer, resulttype) @@ -227,6 +227,12 @@ returncode = self.wait() return returncode + def is_spam(self, fnname, *args): + # To hide the spamming amounts of reads and writes to stdin and stdout + # in interactive sessions + return (fnname in ('ll_os.ll_os_read', 'll_os.ll_os_write') and + args[0] in (0, 1)) + def handle_message(self, fnname, *args): if '__' in fnname: raise ValueError("unsafe fnname") From arigo at codespeak.net Fri Sep 7 14:15:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 14:15:23 +0200 (CEST) Subject: [pypy-svn] r46396 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907121523.4BF4E810C@code0.codespeak.net> Author: arigo Date: Fri Sep 7 14:15:22 2007 New Revision: 46396 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py Log: Hide stderr spam too. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 14:15:22 2007 @@ -231,7 +231,7 @@ # To hide the spamming amounts of reads and writes to stdin and stdout # in interactive sessions return (fnname in ('ll_os.ll_os_read', 'll_os.ll_os_write') and - args[0] in (0, 1)) + args[0] in (0, 1, 2)) def handle_message(self, fnname, *args): if '__' in fnname: From arigo at codespeak.net Fri Sep 7 14:35:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 14:35:28 +0200 (CEST) Subject: [pypy-svn] r46397 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907123528.899FB8119@code0.codespeak.net> Author: arigo Date: Fri Sep 7 14:35:25 2007 New Revision: 46397 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py Log: Turn this AttributeError into a RuntimeError, which can be reported to the subprocess. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 14:35:25 2007 @@ -236,7 +236,10 @@ def handle_message(self, fnname, *args): if '__' in fnname: raise ValueError("unsafe fnname") - handler = getattr(self, 'do_' + fnname.replace('.', '__')) + try: + handler = getattr(self, 'do_' + fnname.replace('.', '__')) + except AttributeError: + raise RuntimeError("no handler for this function") resulttype = getattr(handler, 'resulttype', None) return handler(*args), resulttype From arigo at codespeak.net Fri Sep 7 15:11:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 15:11:57 +0200 (CEST) Subject: [pypy-svn] r46398 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907131157.2F5858105@code0.codespeak.net> Author: arigo Date: Fri Sep 7 15:11:56 2007 New Revision: 46398 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py Log: * Don't count the time waiting for interactive input in the timeout. As a result, cannot timeout any more when we are there, which allows a hack to be removed. * Don't show a traceback with internal details of the controller for the case where the child goes away while we were processing a request. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Fri Sep 7 15:11:56 2007 @@ -19,6 +19,7 @@ 'result': ((34,), False), 'exception': ((34,), False), 'vpath': ((35,), False), + 'timeout': ((1, 31), True), } log = py.log.Producer("sandlib") @@ -117,6 +118,7 @@ env={}) self.popenlock = None self.currenttimeout = None + self.currentlyidlefrom = None def withlock(self, function, *args, **kwds): lock = self.popenlock @@ -136,6 +138,8 @@ def _waiting_thread(): while True: + while self.currentlyidlefrom is not None: + time.sleep(1) # can't timeout while idle t = self.currenttimeout if t is None: return # cancelled @@ -143,13 +147,14 @@ if delay <= 0.0: break # expired! time.sleep(min(delay*1.001, 1)) + log.timeout("timeout!") self.kill() - if interrupt_main: - if hasattr(os, 'kill'): - import signal - os.kill(os.getpid(), signal.SIGINT) - else: - thread.interrupt_main() + #if interrupt_main: + # if hasattr(os, 'kill'): + # import signal + # os.kill(os.getpid(), signal.SIGINT) + # else: + # thread.interrupt_main() def _settimeout(): need_new_thread = self.currenttimeout is None @@ -164,6 +169,20 @@ def canceltimeout(self): """Cancel the current timeout.""" self.currenttimeout = None + self.currentlyidlefrom = None + + def enter_idle(self): + self.currentlyidlefrom = time.time() + + def leave_idle(self): + def _postpone_timeout(): + t = self.currentlyidlefrom + if t is not None and self.currenttimeout is not None: + self.currenttimeout += time.time() - t + try: + self.withlock(_postpone_timeout) + finally: + self.currentlyidlefrom = None def poll(self): returncode = self.withlock(self.popen.poll) @@ -221,9 +240,16 @@ else: if self.debug and not self.is_spam(fnname, *args): log.result(shortrepr(answer)) - write_message(child_stdin, 0) # error code - 0 for ok - write_message(child_stdin, answer, resulttype) - child_stdin.flush() + try: + write_message(child_stdin, 0) # error code - 0 for ok + write_message(child_stdin, answer, resulttype) + child_stdin.flush() + except (IOError, OSError): + # likely cause: subprocess is dead, child_stdin closed + if self.poll() is not None: + break + else: + raise returncode = self.wait() return returncode @@ -296,7 +322,14 @@ elif self._input.isatty(): # don't wait for all 'size' chars if reading from a tty, # to avoid blocking. Instead, stop after reading a line. - return self._input.readline(size) + + # For now, waiting at the interactive console is the + # only time that counts as idle. + self.enter_idle() + try: + return self._input.readline(size) + finally: + self.leave_idle() else: return self._input.read(size) raise OSError("trying to read from fd %d" % (fd,)) @@ -312,6 +345,13 @@ # let's allow access to the real time def do_ll_time__ll_time_sleep(self, seconds): + # regularly check for timeouts that could have killed the + # subprocess + while seconds > 5.0: + time.sleep(5.0) + seconds -= 5.0 + if self.poll() is not None: # subprocess finished? + return time.sleep(seconds) def do_ll_time__ll_time_time(self): From arigo at codespeak.net Fri Sep 7 16:11:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 16:11:45 +0200 (CEST) Subject: [pypy-svn] r46404 - pypy/dist/pypy/module/_weakref Message-ID: <20070907141145.A077F810F@code0.codespeak.net> Author: arigo Date: Fri Sep 7 16:11:44 2007 New Revision: 46404 Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py Log: Forbid user-created weakrefs in a sandbox translation to avoid segfault-related issues. Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Fri Sep 7 16:11:44 2007 @@ -122,9 +122,16 @@ self.w_hash = self.space.hash(w_obj) return self.w_hash +def check(space): + if space.config.translation.sandbox: + msg = "weakrefs are disabled in a sandbox translation at the moment" + raise OperationError(space.w_RuntimeError, + space.wrap(msg)) + def descr__new__weakref(space, w_subtype, w_obj, w_callable=None): lifeline = w_obj.getweakref() if lifeline is None: + check(space) lifeline = WeakrefLifeline() w_obj.setweakref(space, lifeline) return lifeline.get_weakref(space, w_subtype, w_obj, w_callable) @@ -196,6 +203,7 @@ is about to be finalized.""" lifeline = w_obj.getweakref() if lifeline is None: + check(space) lifeline = WeakrefLifeline() w_obj.setweakref(space, lifeline) return lifeline.get_proxy(space, w_obj, w_callable) From antocuni at codespeak.net Fri Sep 7 19:15:01 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Sep 2007 19:15:01 +0200 (CEST) Subject: [pypy-svn] r46407 - in pypy/dist/pypy/translator: cli jvm jvm/test oosupport Message-ID: <20070907171501.50CF580FA@code0.codespeak.net> Author: antocuni Date: Fri Sep 7 19:14:59 2007 New Revision: 46407 Modified: pypy/dist/pypy/translator/cli/constant.py pypy/dist/pypy/translator/jvm/constant.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/test/test_constant.py pypy/dist/pypy/translator/oosupport/constant.py pypy/dist/pypy/translator/oosupport/metavm.py Log: Split the methods that initialize the constants, if they become too large. Modified: pypy/dist/pypy/translator/cli/constant.py ============================================================================== --- pypy/dist/pypy/translator/cli/constant.py (original) +++ pypy/dist/pypy/translator/cli/constant.py Fri Sep 7 19:14:59 2007 @@ -294,11 +294,11 @@ return True return False - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): # special case: dict of void, ignore the values if self._check_for_void_dict(gen): return - return super(CLIDictMixin, self).initialize_data(gen) + return super(CLIDictMixin, self).initialize_data(constgen, gen) # ______________________________________________________________________ # Constant Classes @@ -390,7 +390,7 @@ gen.ilasm.new('instance void class %s::.ctor(object, native int)' % self.delegate_type) self.db.const_count.inc('StaticMethod') - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): return @@ -403,7 +403,7 @@ def get_type(self, include_class=True): return 'class ' + WEAKREF - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): if self.value is not None: push_constant(self.db, self.value._TYPE, self.value, gen) gen.ilasm.call_method( Modified: pypy/dist/pypy/translator/jvm/constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/constant.py (original) +++ pypy/dist/pypy/translator/jvm/constant.py Fri Sep 7 19:14:59 2007 @@ -17,6 +17,8 @@ class JVMConstantGenerator(BaseConstantGenerator): + MAX_INSTRUCTION_COUNT = 20000 + def __init__(self, db): BaseConstantGenerator.__init__(self, db) self.num_constants = 0 @@ -112,6 +114,13 @@ # in JVM, this is done first, in gen_constants() return + def _consider_split_current_function(self, gen): + if gen.get_instruction_count() >= self.MAX_INSTRUCTION_COUNT: + const = self.current_const + gen.pop(const.value._TYPE) + self._new_step(gen) + self._push_constant_during_init(gen, const) + def _declare_step(self, gen, stepnum): self.step_classes.append(JvmClassType('pypy.ConstantInit_%d' % stepnum)) gen.begin_class(self.step_classes[-1], jObject) @@ -149,7 +158,7 @@ else: gen.push_null(jObject) - def initialize_data(self, ilasm): + def initialize_data(self, constgen, gen): return class JVMCustomDictConst(CustomDictConst): @@ -191,7 +200,7 @@ push_constant(self.db, self.value._TYPE, self.value, gen) gen.finalize_cast_ptr_to_weak_address(TYPE) - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): gen.pop(ootype.ROOT) return True Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Fri Sep 7 19:14:59 2007 @@ -1366,3 +1366,5 @@ tryendlbl.jasmin_syntax(), catchlbl.jasmin_syntax())) + def get_instruction_count(self): + return self.curfunc.instr_counter Modified: pypy/dist/pypy/translator/jvm/test/test_constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_constant.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_constant.py Fri Sep 7 19:14:59 2007 @@ -3,6 +3,4 @@ from pypy.translator.jvm.test.runtest import JvmTest class TestConstant(BaseTestConstant, JvmTest): - - def test_many_constants(self): - py.test.skip('Initializing a large constant list generates a function too large for the JVM limits') + pass Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Fri Sep 7 19:14:59 2007 @@ -265,7 +265,8 @@ self._consider_step(gen) gen.add_comment("Constant: %s" % const.name) self._push_constant_during_init(gen, const) - if not const.initialize_data(gen): + self.current_const = const + if not const.initialize_data(self, gen): gen.pop(const.OOTYPE()) def _consider_step(self, gen): @@ -273,11 +274,23 @@ start a new step every so often to ensure the initialization functions don't get too large and upset mono or the JVM or what have you. """ - if (self._all_counter % MAX_CONST_PER_STEP) == 0: - self._end_step(gen) - self._declare_step(gen, self._step_counter) # open the next step + if self._all_counter % MAX_CONST_PER_STEP == 0: + self._new_step(gen) self._all_counter += 1 + def _consider_split_current_function(self, gen): + """ + Called during constant initialization; if the backend thinks + the current function is too large, it can close it and open a + new one, pushing again the constant on the stack. The default + implementatio does nothing. + """ + pass + + def _new_step(self, gen): + self._end_step(gen) + self._declare_step(gen, self._step_counter) # open the next step + def _end_step(self, gen): """ Ends the current step if one has begun. """ if self._all_counter != 0: @@ -393,7 +406,7 @@ assert not self.is_null() gen.new(self.value._TYPE) - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): """ Initializes the internal data. Begins with a pointer to the constant on the stack. Normally returns something @@ -441,7 +454,7 @@ value = self.value._items[f_name] self._record_const_if_complex(FIELD_TYPE, value) - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): assert not self.is_null() SELFTYPE = self.value._TYPE for f_name, (FIELD_TYPE, f_default) in self.value._TYPE._fields.iteritems(): @@ -509,7 +522,7 @@ return const_list - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): assert not self.is_null() # Get a list of all the constants we'll need to initialize. @@ -525,6 +538,7 @@ # Store each of our fields in the sorted order for FIELD_TYPE, INSTANCE, name, value in const_list: + constgen._consider_split_current_function(gen) gen.dup(SELFTYPE) push_constant(self.db, FIELD_TYPE, value, gen) gen.set_field(INSTANCE, name) @@ -550,7 +564,7 @@ INSTANCE = self.value._INSTANCE gen.getclassobject(INSTANCE) - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): pass # ______________________________________________________________________ @@ -596,7 +610,7 @@ except: return False - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): assert not self.is_null() SELFTYPE = self.value._TYPE ITEMTYPE = self.value._TYPE._ITEMTYPE @@ -607,6 +621,7 @@ # set each item in the list using the OOTYPE methods for idx, item in enumerate(self.value._list): + constgen._consider_split_current_function(gen) gen.dup(SELFTYPE) push_constant(self.db, ootype.Signed, idx, gen) push_constant(self.db, ITEMTYPE, item, gen) @@ -631,7 +646,7 @@ self._record_const_if_complex(self.value._TYPE._KEYTYPE, key) self._record_const_if_complex(self.value._TYPE._VALUETYPE, value) - def initialize_data(self, gen): + def initialize_data(self, constgen, gen): assert not self.is_null() SELFTYPE = self.value._TYPE KEYTYPE = self.value._TYPE._KEYTYPE @@ -644,6 +659,7 @@ return for key, value in self.value._dict.iteritems(): + constgen._consider_split_current_function(gen) gen.dup(SELFTYPE) gen.add_comment(' key=%r value=%r' % (key,value)) push_constant(self.db, KEYTYPE, key, gen) @@ -669,7 +685,7 @@ self.db.pending_function(self.value.graph) self.delegate_type = self.db.record_delegate(self.value._TYPE) - def initialize_data(self, ilasm): + def initialize_data(self, constgen, gen): raise NotImplementedError # ______________________________________________________________________ Modified: pypy/dist/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/metavm.py (original) +++ pypy/dist/pypy/translator/oosupport/metavm.py Fri Sep 7 19:14:59 2007 @@ -195,7 +195,14 @@ value. TYPE will be one of the types enumerated in oosupport.constant.PRIMITIVE_TYPES. value will be its corresponding ootype implementation. """ - raise NotImplementedError + raise NotImplementedError + + def get_instrution_count(self): + """ + Return the number of opcodes in the current function, or -1 + if the backend doesn't care about it. Default is -1 + """ + return -1 class InstructionList(list): def render(self, generator, op): From antocuni at codespeak.net Fri Sep 7 20:08:06 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Sep 2007 20:08:06 +0200 (CEST) Subject: [pypy-svn] r46408 - pypy/dist/pypy/translator/jvm Message-ID: <20070907180806.C91BC80FA@code0.codespeak.net> Author: antocuni Date: Fri Sep 7 20:08:05 2007 New Revision: 46408 Modified: pypy/dist/pypy/translator/jvm/opcodes.py Log: ignore this opcode Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Fri Sep 7 20:08:05 2007 @@ -68,6 +68,7 @@ 'cast_ptr_to_weakadr': [CastPtrToWeakAddress], 'cast_weakadr_to_ptr': CastWeakAddressToPtr, 'gc__collect': jvmgen.SYSTEMGC, + 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, 'debug_assert': [], # TODO: implement? From arigo at codespeak.net Fri Sep 7 20:19:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Sep 2007 20:19:13 +0200 (CEST) Subject: [pypy-svn] r46409 - pypy/dist/pypy/translator/sandbox Message-ID: <20070907181913.65C9980D0@code0.codespeak.net> Author: arigo Date: Fri Sep 7 20:19:13 2007 New Revision: 46409 Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py Log: Precision in the docstring: --heapsize won't work out of the box with other sandboxed executables than PyPy. Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Fri Sep 7 20:19:13 2007 @@ -10,7 +10,7 @@ which is the virtual current dir (always read-only for now) --heapsize=N limit memory usage to N bytes, or kilo- mega- giga-bytes with the 'k', 'm' or 'g' suffix respectively. - ATM this only works if the sandboxed executable uses Boehm. + ATM this only works with PyPy translated with Boehm. --timeout=N limit execution time to N (real-time) seconds. """ From antocuni at codespeak.net Fri Sep 7 20:28:15 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Sep 2007 20:28:15 +0200 (CEST) Subject: [pypy-svn] r46410 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070907182815.83E8780DC@code0.codespeak.net> Author: antocuni Date: Fri Sep 7 20:28:15 2007 New Revision: 46410 Modified: pypy/dist/pypy/translator/jvm/src/pypy/CustomDict.java pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Log: comment out some printlns Modified: pypy/dist/pypy/translator/jvm/src/pypy/CustomDict.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/CustomDict.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/CustomDict.java Fri Sep 7 20:28:15 2007 @@ -30,8 +30,8 @@ public CustomDict(final Equals equals, final HashCode hashCode) { this.hashCode = hashCode; this.equals = equals; - System.err.println("Equals: "+equals.getClass()); - System.err.println("HashCode: "+hashCode.getClass()); + //System.err.println("Equals: "+equals.getClass()); + //System.err.println("HashCode: "+hashCode.getClass()); } public class Adapter { @@ -142,4 +142,4 @@ return this.backingMap.values(); } -} \ No newline at end of file +} Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Fri Sep 7 20:28:15 2007 @@ -571,8 +571,8 @@ return -1; int res = haystack.indexOf(needle, start); - System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ - " end="+end+" res="+res); + //System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ + // " end="+end+" res="+res); if (res + needle.length() > end) return -1; return res; @@ -580,8 +580,8 @@ public static int ll_rfind(String haystack, String needle, int start, int end) { int res = haystack.lastIndexOf(needle, end-1); - System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ - " end="+end+" res="+res); + //System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ + // " end="+end+" res="+res); if (res >= start) return res; return -1; @@ -616,8 +616,8 @@ public static int ll_rfind_char(String haystack, char needle, int start, int end) { int res = haystack.lastIndexOf(needle, end-1); - System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ - " end="+end+" res="+res); + //System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ + // " end="+end+" res="+res); if (res >= start) return res; return -1; @@ -802,6 +802,16 @@ return text.length(); } + public static ArrayList ll_os_envitems() + { + return new ArrayList(); + } + + public static String ll_os_getcwd() + { + return "/tmp"; + } + // ---------------------------------------------------------------------- // Exceptions // @@ -869,7 +879,7 @@ } public static void _ll_resize_le(ArrayList self, int length) { - System.err.println("ll_resize_le: self.size()="+self.size()+" length="+length); + //System.err.println("ll_resize_le: self.size()="+self.size()+" length="+length); while (self.size() > length) { self.remove(self.size()-1); } From simonb at codespeak.net Sat Sep 8 00:23:27 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Sat, 8 Sep 2007 00:23:27 +0200 (CEST) Subject: [pypy-svn] r46411 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070907222327.3476A80EA@code0.codespeak.net> Author: simonb Date: Sat Sep 8 00:23:25 2007 New Revision: 46411 Modified: pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: numpy: less code, more features. that's what we like. Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Sat Sep 8 00:23:25 2007 @@ -4,12 +4,13 @@ from pypy.rpython.rlist import AbstractBaseListRepr from pypy.rpython.rtuple import AbstractTupleRepr from pypy.rpython.error import TyperError -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rtuple from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython.lltypesystem.lltype import \ - GcArray, GcStruct, Signed, Ptr, Unsigned, Void, FixedSizeArray, Bool,\ + GcArray, GcStruct, Number, Primitive, Signed, Ptr, Unsigned, Void, FixedSizeArray, Bool,\ GcForwardReference, malloc, direct_arrayitems, direct_ptradd, nullptr +from pypy.rpython.lltypesystem.rffi import cast from pypy.rpython.lltypesystem.rtuple import TupleRepr from pypy.annotation.model import SomeObject, SomeInteger from pypy.rpython.numpy.aarray import SomeArray @@ -98,9 +99,22 @@ it.coordinates[i] = 0 ll_iter_reset._always_inline_ = True - def ll_iter_new(ITER, ao, iter_reset=ll_iter_reset): + def ll_iter_new(ITER, ao, target_ao, iter_reset, iter_broadcast_to_shape): assert ao.dataptr - assert ao.ndim == ndim + # Suffix of ao.shape must match target_ao.shape + # (suffix starts at the first non-1 entry in ao.shape.) + # ao.shape must be no longer than target_ao.shape. + assert ao.ndim <= ndim + assert target_ao.ndim == ndim + # XX check suffix condition here... ? + broadcast = ao.ndim < ndim + i = 0 + while not broadcast and i < ao.ndim: + if ao.shape[i] == 1 and target_ao.shape[i] > 1: + broadcast = True + i += 1 + if broadcast: + return iter_broadcast_to_shape(ITER, ao, target_ao, iter_reset) it = malloc(ITER) it.nd_m1 = ndim - 1 it.size = ll_mul_list(ao.shape, ndim) @@ -115,7 +129,7 @@ return it ll_iter_new._always_inline_ = True - def ll_iter_broadcast_to_shape(ITER, ao, target_ao, iter_reset=ll_iter_reset): + def ll_iter_broadcast_to_shape(ITER, ao, target_ao, iter_reset): "iterate over but broadcast to the shape of " assert target_ao.ndim == ndim delta = j = ndim - ao.ndim @@ -143,6 +157,9 @@ return it ll_iter_broadcast_to_shape._always_inline_ = True + # XXX + # I think this is the only function that needs to be + # generated-per-ndim: def ll_iter_next(it): it.index += 1 for i in unroll_ndim_rev: @@ -164,6 +181,8 @@ def ll_array_set(it0, it1, iter_next): + if it0.size == 0: + return # empty LHS.. assert it0.size == it1.size while it0.index < it0.size: it0.dataptr[0] = it1.dataptr[0] @@ -187,25 +206,30 @@ self.lowleveltype = self.ITER class ArrayRepr(Repr): + def make_types(cls, ndim, ITEM): + DATA_PTR = Ptr(FixedSizeArray(ITEM, 1)) + ITEMARRAY = GcArray(ITEM, hints={'nolength':True}) + INDEXARRAY = FixedSizeArray(NPY_INTP, ndim) + STRUCT = GcStruct("array", + ("data", Ptr(ITEMARRAY)), # pointer to raw data buffer + ("dataptr", DATA_PTR), # pointer to first element + ("ndim", Signed), # number of dimensions + ("shape", INDEXARRAY), # size in each dimension + ("strides", INDEXARRAY), # elements to jump to get to the + # next element in each dimension + ) + ARRAY = Ptr(STRUCT) + return ARRAY, INDEXARRAY + make_types = classmethod(make_types) + def __init__(self, rtyper, s_array): self.s_array = s_array self.s_value = s_array.get_item_type() self.ndim = s_array.ndim - self.item_repr = rtyper.getrepr(self.s_value) # XX rename r_item XX - self.ITEM = self.item_repr.lowleveltype - ITEMARRAY = GcArray(self.ITEM, hints={'nolength':True}) - self.INDEXARRAY = FixedSizeArray(NPY_INTP, self.ndim) + self.r_item = rtyper.getrepr(self.s_value) + self.ITEM = self.r_item.lowleveltype self.itemsize = sizeof(self.ITEM) - DATA_PTR = Ptr(FixedSizeArray(self.ITEM, 1)) - self.STRUCT = GcStruct("array", - ("data", Ptr(ITEMARRAY)), # pointer to raw data buffer - ("dataptr", DATA_PTR), # pointer to first element - ("ndim", Signed), # number of dimensions - ("shape", self.INDEXARRAY), # size in each dimension - ("strides", self.INDEXARRAY), # elements to jump to get to the - # next element in each dimension - ) - self.ARRAY = Ptr(self.STRUCT) + self.ARRAY, self.INDEXARRAY = self.make_types(self.ndim, self.ITEM) self.lowleveltype = self.ARRAY self.ITER = ARRAY_ITER(self.ARRAY, self.INDEXARRAY) @@ -283,96 +307,6 @@ cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) return hop.gendirectcall(ll_add, cARRAY, v_arr1, v_arr2) - -#class __extend__(pairtype(ArrayRepr, Repr)): # <------ USE THIS ?? -class __extend__(pairtype(ArrayRepr, IntegerRepr)): - def rtype_setitem((r_array, r_int), hop): - assert r_array.ndim == 1, "NotImplemented" - v_array, v_index, v_item = hop.inputargs(r_array, Signed, r_array.item_repr) - return hop.gendirectcall(ll_setitem1, v_array, v_index, v_item) - - def rtype_getitem((r_array, r_int), hop): - assert r_array.ndim == 1, "NotImplemented" - v_array, v_index = hop.inputargs(r_array, Signed) - return hop.gendirectcall(ll_getitem1, v_array, v_index) - -def gen_get_view_slc(r_array, r_slc, hop): # XX method on the pair type ? - ndim = r_array.ndim - rslice = hop.rtyper.type_system.rslice - def ll_get_view_slc(ARRAY, ao, slc): - array = ll_allocate(ARRAY, ndim) - dataptr = direct_arrayitems(ao.data) - src_i = 0 - tgt_i = 0 - if r_slc == rslice.startonly_slice_repr: - start = slc - size = ao.shape[src_i] - if start > size: - start = size - size -= start - dataptr = direct_ptradd(dataptr, start*ao.strides[src_i]) - array.shape[tgt_i] = size - array.strides[tgt_i] = ao.strides[src_i] - tgt_i += 1 - elif r_slc == rslice.startstop_slice_repr: - start = slc.start - stop = slc.stop - size = ao.shape[src_i] - if start > size: - start = size - dataptr = direct_ptradd(dataptr, start*ao.strides[src_i]) - if stop < size: - size = stop - size -= start - if size < 0: - size = 0 - array.shape[tgt_i] = size - array.strides[tgt_i] = ao.strides[src_i] - tgt_i += 1 - else: - assert 0 - src_i += 1 - # consume the rest of ndim as if we found more slices - while tgt_i < ndim: - array.shape[tgt_i] = ao.shape[src_i] - array.strides[tgt_i] = ao.strides[src_i] - tgt_i += 1 - src_i += 1 - assert tgt_i == ndim - array.dataptr = dataptr - array.data = ao.data # keep a ref - return array - return ll_get_view_slc - -class __extend__(pairtype(ArrayRepr, AbstractSliceRepr)): - def rtype_setitem((r_array, r_slc), hop): - r_item = hop.args_r[2] - v_array, v_slc, v_item = hop.inputargs(r_array, r_slc, r_item) - cITER = hop.inputconst(Void, r_array.ITER.TO) - cARRAY = hop.inputconst(Void, r_array.ARRAY.TO) - iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(r_array.ndim) - cnext = hop.inputconst(Void, iter_next) - creset = hop.inputconst(Void, iter_reset) -## Blech... it would be nice to reuse gen_get_view -## r_tuple = TupleRepr(hop.rtyper, [r_item]) # XX how to get this from rtyper ? -## get_view = gen_get_view(r_array, r_tuple, hop) -## # make a v_tuple here... -## v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_tuple) - get_view = gen_get_view_slc(r_array, r_slc, hop) - v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_slc) - v_it0 = hop.gendirectcall(iter_new, cITER, v_view, creset) - if isinstance(r_item, ArrayRepr): - if r_array.ndim == r_item.ndim: - v_it1 = hop.gendirectcall(iter_new, cITER, v_item, creset) - else: - v_it1 = hop.gendirectcall(iter_broadcast, cITER, v_item, v_array, creset) - assert r_array.ndim >= r_item.ndim - return hop.gendirectcall(ll_array_set, v_it0, v_it1, cnext) - elif isinstance(r_item, FloatRepr): - # setitem from scalar - return hop.gendirectcall(ll_array_set1, v_item, v_it0, cnext) - else: - raise TypeError("can't setitem from %s"%r_item) def gen_getset_item(ndim): unrolling_dims = unrolling_iterable(range(ndim)) @@ -449,11 +383,52 @@ return array return ll_get_view +def convert_int_to_tuple(r_int, v_int, llops): + # int -> (int,) + r_tuple = TupleRepr(llops.rtyper, [r_int]) # XX get this from rtyper cache ? + v_tuple = rtuple.newtuple(llops, r_tuple, [v_int]) + return r_tuple, v_tuple + +def convert_slice_to_tuple(r_slc, v_slc, llops): + # slice -> (slice,) + r_tuple = TupleRepr(llops.rtyper, [r_slc]) # XX get this from rtyper cache ? + v_tuple = rtuple.newtuple(llops, r_tuple, [v_slc]) + return r_tuple, v_tuple + +def convert_list_to_array(r_list, v_list, llops): + # [...] -> array([...]) + from pypy.rpython.rmodel import inputconst + ITEM = r_list.item_repr.lowleveltype + ARRAY, _ = ArrayRepr.make_types(1, ITEM) + cARRAY = inputconst(Void, ARRAY.TO) + v_array = llops.gendirectcall(ll_build_from_list, cARRAY, v_list) # XX does a copy :P + #v_array = llops.gendirectcall(ll_build_alias_to_list, cARRAY, v_list) # nice idea... + return v_array + +def convert_scalar_to_array(ITEM, v_item, llops): + # x -> array([x]) + from pypy.rpython.rmodel import inputconst + ARRAY, _ = ArrayRepr.make_types(1, ITEM) + cARRAY = inputconst(Void, ARRAY.TO) + cITEM = inputconst(Void, ITEM) + v_casted = llops.genop("cast_primitive", [v_item], ITEM) + v_array = llops.gendirectcall(ll_build_from_scalar, cARRAY, v_casted) + return v_array + +class __extend__(pairtype(ArrayRepr, Repr)): + def rtype_getitem((r_array, r_key), hop): + + v_array, v_key = hop.inputargs(r_array, r_key) + if isinstance(r_key, IntegerRepr): + r_tuple, v_tuple = convert_int_to_tuple(r_key, v_key, hop.llops) + elif isinstance(r_key, AbstractSliceRepr): + r_tuple, v_tuple = convert_slice_to_tuple(r_key, v_key, hop.llops) + elif isinstance(r_key, TupleRepr): + r_tuple, v_tuple = r_key, v_key + else: + raise TyperError("bad key: %s"%r_key) -class __extend__(pairtype(ArrayRepr, AbstractTupleRepr)): - def rtype_getitem((r_array, r_tpl), hop): - v_array, v_tuple = hop.inputargs(r_array, r_tpl) - ndim = get_view_ndim(r_array, r_tpl) + ndim = get_view_ndim(r_array, r_tuple) if ndim == 0: # return a scalar cARRAY = hop.inputconst(Void, r_array.ARRAY.TO) @@ -463,12 +438,20 @@ ARRAY = r_result.ARRAY assert dim_of_ARRAY(ARRAY) == ndim cARRAY = hop.inputconst(Void, ARRAY.TO) - ll_get_view = gen_get_view(r_array, r_tpl, hop) + ll_get_view = gen_get_view(r_array, r_tuple, hop) return hop.gendirectcall(ll_get_view, cARRAY, v_array, v_tuple) - def rtype_setitem((r_array, r_tuple), hop): + def rtype_setitem((r_array, r_key), hop): r_item = hop.args_r[2] - v_array, v_tuple, v_item = hop.inputargs(r_array, r_tuple, r_item) + v_array, v_key, v_item = hop.inputargs(r_array, r_key, r_item) + if isinstance(r_key, IntegerRepr): + r_tuple, v_tuple = convert_int_to_tuple(r_key, v_key, hop.llops) + elif isinstance(r_key, AbstractSliceRepr): + r_tuple, v_tuple = convert_slice_to_tuple(r_key, v_key, hop.llops) + elif isinstance(r_key, TupleRepr): + r_tuple, v_tuple = r_key, v_key + else: + raise TyperError("bad key: %s"%r_key) ndim = get_view_ndim(r_array, r_tuple) assert len(r_tuple.items_r) <= r_array.ndim if ndim == 0: @@ -477,7 +460,7 @@ cARRAY = hop.inputconst(Void, r_array.ARRAY.TO) get_item, set_item = gen_getset_item(r_array.ndim) return hop.gendirectcall(set_item, cARRAY, v_array, v_tuple, v_item) - elif isinstance(r_item, ArrayRepr): + else: s_view = SomeArray(r_array.s_array.typecode, ndim) r_view = hop.rtyper.getrepr(s_view) cARRAY = hop.inputconst(Void, r_view.ARRAY.TO) @@ -485,17 +468,24 @@ v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_tuple) iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(ndim) creset = hop.inputconst(Void, iter_reset) + cbroadcast = hop.inputconst(Void, iter_broadcast) cnext = hop.inputconst(Void, iter_next) cITER = hop.inputconst(Void, r_view.ITER.TO) - v_it0 = hop.gendirectcall(iter_new, cITER, v_view, creset) - assert r_item.ndim <= ndim - if ndim == r_item.ndim: - v_it1 = hop.gendirectcall(iter_new, cITER, v_item, creset) + v_it0 = hop.gendirectcall(iter_new, cITER, v_view, v_view, creset, cbroadcast) + if isinstance(r_item, ArrayRepr): + source_ndim = r_item.ndim + elif isinstance(r_item.lowleveltype, Primitive): + # "broadcast" a scalar + v_item = convert_scalar_to_array(r_view.ITEM, v_item, hop.llops) + source_ndim = 1 + elif isinstance(r_item, AbstractBaseListRepr): + v_item = convert_list_to_array(r_item, v_item, hop.llops) + source_ndim = 1 else: - v_it1 = hop.gendirectcall(iter_broadcast, cITER, v_item, v_view, creset) + raise TypeError("bad item: %s"%r_item) + assert source_ndim <= ndim + v_it1 = hop.gendirectcall(iter_new, cITER, v_item, v_view, creset, cbroadcast) return hop.gendirectcall(ll_array_set, v_it0, v_it1, cnext) - else: - assert 0 class __extend__(pairtype(ArrayRepr, ArrayRepr)): @@ -506,8 +496,8 @@ def convert_from_to((r_lst, r_array), v, llops): if r_lst.listitem is None: return NotImplemented - if r_lst.item_repr != r_array.item_repr: - assert 0, (r_lst, r_array.item_repr) + if r_lst.item_repr != r_array.r_item: + assert 0, (r_lst, r_array.r_item) return NotImplemented cARRAY = inputconst(lltype.Void, r_array.lowleveltype.TO) return llops.gendirectcall(ll_build_from_list, cARRAY, v) @@ -545,6 +535,26 @@ array.dataptr = direct_arrayitems(array.data) return array +def ll_build_alias_to_list(ARRAY, lst): + # This should only be used for temporary calculations + size = lst.ll_length() + array = ll_allocate(ARRAY, 1) + array.shape[0] = size + array.strides[0] = 1 + # Well.. this doesn't work (because array.data has nolength ?) + array.data = lst.ll_items() + array.dataptr = direct_arrayitems(array.data) + return array + +def ll_build_from_scalar(ARRAY, value): + array = ll_allocate(ARRAY, 1) + array.shape[0] = 1 + array.strides[0] = 1 + array.data = malloc(ARRAY.data.TO, 1) + array.dataptr = direct_arrayitems(array.data) + array.data[0] = value + return array + def ll_build_alias(ARRAY, ao): array = ll_allocate(ARRAY, ao.ndim) array.data = ao.data # alias data @@ -554,11 +564,11 @@ array.dataptr = ao.dataptr return array -def ll_setitem1(l, index, item): - l.data[index] = item +def ll_setitem1(array, index, item): + array.data[index] = item -def ll_getitem1(l, index): - return l.data[index] +def ll_getitem1(array, index): + return array.data[index] def ll_add(ARRAY, a1, a2): size = a1.shape[0] Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Sat Sep 8 00:23:25 2007 @@ -354,6 +354,16 @@ assert interpret(f, [0, 0]) == 0 assert interpret(f, [3, 4]) == 12 + def test_specialize_list_rhs(self): + def f(): + a = numpy.zeros((3,4), dtype='i') + a[:] = [3,4,4,2] + return a + res = interpret(f, []) + data = [3,4,4,2]*3 + for i in range(len(data)): + assert res.dataptr[i] == data[i] + def test_specialize_slice_1_0(self): def f(): a = numpy.zeros((12,), dtype='i') @@ -369,7 +379,7 @@ assert res.dataptr[i] == data[i] def test_specialize_slice_1_1(self): - py.test.skip('this involves a runtime test to see if we need a broadcast iterator') + # This involves a runtime test to see if we need a broadcast iterator def f(): a = numpy.zeros((6,), dtype='i') a[:2] = numpy.array([1]) @@ -381,10 +391,9 @@ assert res.dataptr[i] == data[i] def test_specialize_slice_2_0(self): - py.test.skip('not implemented') def f(): a = numpy.zeros((12,), dtype='i').reshape((3,4)) - a[:2, 0] = 1 + a[:2, :1] = 1 return a res = interpret(f, []) data = [1,0,0,0,1,0,0,0,0,0,0,0] @@ -467,7 +476,6 @@ assert res.dataptr[i] == data[i] def test_malloc_remove(self): - py.test.skip('this test requires _always_inline_ magic hook') def f(): a = numpy.empty((3,), dtype='i') b = numpy.array([5,55,555], dtype='i') @@ -480,8 +488,8 @@ r.specialize() from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) - from pypy.rpython.numpy.rarray import ll_array_unary_op - graph = graphof(t, ll_array_unary_op) + from pypy.rpython.numpy.rarray import ll_array_set + graph = graphof(t, ll_array_set) #graph.show() from pypy.translator.backendopt.test.test_malloc import TestLLTypeMallocRemoval TestLLTypeMallocRemoval.check_malloc_removed(graph) From simonb at codespeak.net Sat Sep 8 03:53:55 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Sat, 8 Sep 2007 03:53:55 +0200 (CEST) Subject: [pypy-svn] r46412 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070908015355.6E02580E5@code0.codespeak.net> Author: simonb Date: Sat Sep 8 03:53:54 2007 New Revision: 46412 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: hacking on arithmetic. broke it. Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Sat Sep 8 03:53:54 2007 @@ -9,6 +9,23 @@ import numpy +numpy_typedict = { + (SomeInteger, rffi.r_signedchar) : 'b', + (SomeInteger, rffi.r_short) : 'h', + (SomeInteger, rffi.r_int) : 'i', + (SomeInteger, rffi.r_long) : 'l', + (SomeInteger, int) : 'l', + (SomeInteger, rffi.r_longlong) : 'q', + (SomeInteger, rffi.r_uchar) : 'B', + (SomeInteger, rffi.r_ushort) : 'H', + (SomeInteger, rffi.r_uint) : 'I', + (SomeInteger, rffi.r_ulong) : 'L', + (SomeInteger, rffi.r_ulonglong) : 'Q', + #(SomeFloat, float) : 'f', + (SomeFloat, float) : 'd', +} +valid_typecodes='bhilqBHILQd' + class SomeArray(SomeObject): """Stands for an object from the numpy module.""" typecode_to_item = { @@ -25,12 +42,16 @@ #'f' : SomeFloat(), # XX single precision float XX 'd' : SomeFloat(), } + def __init__(self, typecode, ndim=1): if not typecode in self.typecode_to_item: raise AnnotatorError("bad typecode: %r"%typecode) self.typecode = typecode self.ndim = ndim + def get_one_dim(self): + return SomeArray(self.typecode) + def can_be_none(self): return True @@ -76,11 +97,34 @@ break if typecode is None: raise AnnotatorError() + ndim = max(s_arr1.ndim, s_arr2.ndim) return SomeArray(typecode, s_arr1.ndim) # union ? sub = mul = div = add +class __extend__(pairtype(SomeArray, SomeFloat)): + def add((s_arr, s_flt)): + item = s_arr.get_item_type() + typecode = None + if float in (item.knowntype, s_flt.knowntype): + typecode = 'd' + else: + item_knowntype = rarithmetic.compute_restype(item.knowntype, s_flt.knowntype) + for typecode, s_item in SomeArray.typecode_to_item.items(): + if s_item.knowntype == item_knowntype: + break + if typecode is None: + raise AnnotatorError() + return SomeArray(typecode, s_arr.ndim) + # union ? + sub = mul = div = add + +class __extend__(pairtype(SomeFloat, SomeArray)): + def add((s_flt, s_arr)): + return pair(s_arr, s_flt).add() + # union ? + sub = mul = div = add class __extend__(pairtype(SomeArray, SomeTuple)): def get_leftover_dim((s_array, s_index)): @@ -133,23 +177,6 @@ s_tuple = SomeTuple([s_index]) return pair(s_array, s_tuple).getitem() -numpy_typedict = { - (SomeInteger, rffi.r_signedchar) : 'b', - (SomeInteger, rffi.r_short) : 'h', - (SomeInteger, rffi.r_int) : 'i', - (SomeInteger, rffi.r_long) : 'l', - (SomeInteger, int) : 'l', - (SomeInteger, rffi.r_longlong) : 'q', - (SomeInteger, rffi.r_uchar) : 'B', - (SomeInteger, rffi.r_ushort) : 'H', - (SomeInteger, rffi.r_uint) : 'I', - (SomeInteger, rffi.r_ulong) : 'L', - (SomeInteger, rffi.r_ulonglong) : 'Q', - #(SomeFloat, float) : 'f', - (SomeFloat, float) : 'd', -} -valid_typecodes='bhilqBHILQfd' - class ArrayCallEntry(ExtRegistryEntry): _about_ = numpy.array Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Sat Sep 8 03:53:54 2007 @@ -18,6 +18,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.annotation import listdef from pypy.rpython.memory.lltypelayout import sizeof +from pypy.rlib.objectmodel import debug_assert def gen_build_from_shape(ndim, zero=False): unrolling_dims = unrolling_iterable(reversed(range(ndim))) @@ -104,8 +105,8 @@ # Suffix of ao.shape must match target_ao.shape # (suffix starts at the first non-1 entry in ao.shape.) # ao.shape must be no longer than target_ao.shape. - assert ao.ndim <= ndim - assert target_ao.ndim == ndim + debug_assert(ao.ndim <= ndim, "ao.ndim <= ndim") + debug_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim") # XX check suffix condition here... ? broadcast = ao.ndim < ndim i = 0 @@ -131,7 +132,7 @@ def ll_iter_broadcast_to_shape(ITER, ao, target_ao, iter_reset): "iterate over but broadcast to the shape of " - assert target_ao.ndim == ndim + debug_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim") delta = j = ndim - ao.ndim shape = target_ao.shape for i in range(ao.ndim): @@ -183,7 +184,7 @@ def ll_array_set(it0, it1, iter_next): if it0.size == 0: return # empty LHS.. - assert it0.size == it1.size + debug_assert(it0.size == it1.size, "it0.size == it1.size") while it0.index < it0.size: it0.dataptr[0] = it1.dataptr[0] iter_next(it0) @@ -194,6 +195,15 @@ it.dataptr[0] = value iter_next(it) +def ll_array_add(it0, it1, it2, iter_next): + debug_assert(it0.size == it1.size, "it0.size == it1.size") + debug_assert(it1.size == it2.size, "it0.size == it1.size") + while it0.index < it0.size: + it0.dataptr[0] = it1.dataptr[0] + it2.dataptr[0] + iter_next(it0) + iter_next(it1) + iter_next(it2) + def dim_of_ITER(ITER): return ITER.TO.coordinates.length @@ -303,9 +313,36 @@ class __extend__(pairtype(ArrayRepr, ArrayRepr)): def rtype_add((r_array1, r_array2), hop): - v_arr1, v_arr2 = hop.inputargs(r_array1, r_array2) - cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) - return hop.gendirectcall(ll_add, cARRAY, v_arr1, v_arr2) + v_array1, v_array2 = hop.inputargs(r_array1, r_array2) + r_array0 = hop.r_result + cARRAY = hop.inputconst(Void, r_array0.ARRAY.TO) + # We build a contiguous "result" array + # from the largest of the two args: + v_bigarray = hop.gendirectcall(ll_find_largest, cARRAY, v_array1, v_array2) + v_array0 = hop.gendirectcall(ll_build_like, cARRAY, v_bigarray) + iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(r_array0.ndim) + cITER = hop.inputconst(Void, r_array0.ITER.TO) + creset = hop.inputconst(Void, iter_reset) + cbroadcast = hop.inputconst(Void, iter_broadcast) + cnext = hop.inputconst(Void, iter_next) + v_it0 = hop.gendirectcall(iter_new, cITER, v_array0, v_bigarray, creset, cbroadcast) + v_it1 = hop.gendirectcall(iter_new, cITER, v_array1, v_bigarray, creset, cbroadcast) + v_it2 = hop.gendirectcall(iter_new, cITER, v_array2, v_bigarray, creset, cbroadcast) + return hop.gendirectcall(ll_array_add, v_it0, v_it1, v_it2, cnext) + +class __extend__(pairtype(ArrayRepr, Repr)): + def rtype_add((r_array, r_ob), hop): + assert 0 + v_ob = hop.inputarg(r_ob, 1) + if isinstance(r_ob.lowleveltype, Primitive): + r_item, v_item = convert_scalar_to_array(r_array, v_ob, hop.llops) + +class __extend__(pairtype(Repr, ArrayRepr)): + def rtype_add((r_ob, r_array), hop): + # XX ach! how to get this to work ?? + assert 0 + return pair(r_array, r_ob).rtype_add(hop) + def gen_getset_item(ndim): @@ -377,7 +414,7 @@ array.strides[tgt_i] = ao.strides[src_i] tgt_i += 1 src_i += 1 - assert tgt_i == ndim + debug_assert(tgt_i == ndim, "tgt_i == ndim") array.dataptr = dataptr array.data = ao.data # keep a ref return array @@ -405,15 +442,17 @@ #v_array = llops.gendirectcall(ll_build_alias_to_list, cARRAY, v_list) # nice idea... return v_array -def convert_scalar_to_array(ITEM, v_item, llops): +def convert_scalar_to_array(r_array, v_item, llops): # x -> array([x]) + s_array = r_array.s_array.get_one_dim() + r_array = llops.rtyper.getrepr(s_array) from pypy.rpython.rmodel import inputconst - ARRAY, _ = ArrayRepr.make_types(1, ITEM) - cARRAY = inputconst(Void, ARRAY.TO) - cITEM = inputconst(Void, ITEM) - v_casted = llops.genop("cast_primitive", [v_item], ITEM) +# ARRAY, _ = ArrayRepr.make_types(1, ITEM) + cARRAY = inputconst(Void, r_array.ARRAY.TO) + cITEM = inputconst(Void, r_array.ITEM) + v_casted = llops.genop("cast_primitive", [v_item], r_array.ITEM) v_array = llops.gendirectcall(ll_build_from_scalar, cARRAY, v_casted) - return v_array + return r_array, v_array class __extend__(pairtype(ArrayRepr, Repr)): def rtype_getitem((r_array, r_key), hop): @@ -476,7 +515,7 @@ source_ndim = r_item.ndim elif isinstance(r_item.lowleveltype, Primitive): # "broadcast" a scalar - v_item = convert_scalar_to_array(r_view.ITEM, v_item, hop.llops) + r_item, v_item = convert_scalar_to_array(r_view, v_item, hop.llops) source_ndim = 1 elif isinstance(r_item, AbstractBaseListRepr): v_item = convert_list_to_array(r_item, v_item, hop.llops) @@ -487,7 +526,6 @@ v_it1 = hop.gendirectcall(iter_new, cITER, v_item, v_view, creset, cbroadcast) return hop.gendirectcall(ll_array_set, v_it0, v_it1, cnext) - class __extend__(pairtype(ArrayRepr, ArrayRepr)): def convert_from_to((r_array0, r_array1), v, llops): assert 0 @@ -564,6 +602,29 @@ array.dataptr = ao.dataptr return array +def ll_find_largest(ARRAY, array0, array1): + sz0 = ll_mul_list(array0.shape, array0.ndim) + sz1 = ll_mul_list(array1.shape, array1.ndim) + # XXX + if sz0 > sz1: + return array0 + return array1 + +def ll_build_like(ARRAY, ao): + array = ll_allocate(ARRAY, ao.ndim) + sz = ll_mul_list(ao.shape) + array.data = malloc(ARRAY.data.TO, sz) + array.dataptr = array.data + itemsize = 1 + i = ao.ndim - 1 + while i >= 0: + size = ao.shape[i] + array.shape[i] = size + array.strides[i] = itemsize + itemsize *= size + i -= 1 + return array + def ll_setitem1(array, index, item): array.data[index] = item Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Sat Sep 8 03:53:54 2007 @@ -34,6 +34,7 @@ return my_array[0] class Test_annotation: + def test_annotate_array_access_int(self): t = TranslationContext() a = t.buildannotator() @@ -105,6 +106,7 @@ assert s.ndim == 2 def test_annotate_array_add(self): + py.test.skip("broken ATM") def f(): a1 = numpy.array([1,2]) a2 = numpy.array([6,9]) @@ -116,6 +118,7 @@ assert s.typecode == 'i' def test_annotate_array_add_coerce(self): + py.test.skip("broken ATM") def f(): a1 = numpy.array([1,2]) a2 = numpy.array([6.,9.]) @@ -126,6 +129,18 @@ s = a.build_types(f, []) assert s.typecode == 'd' + def test_annotate_array_add_2(self): + py.test.skip("broken ATM") + def f(): + a = numpy.array([1,2]) + a = a + 3 + return a + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert s.typecode == 'i' + def test_annotate_array_dtype(self): def f(): a1 = numpy.array([1,2], dtype='d') @@ -266,6 +281,7 @@ assert res == 45 def test_specialize_array_add(self): + py.test.skip("broken ATM") def f(): a1 = numpy.array([1.,2.]) a2 = numpy.array([6,9]) @@ -275,6 +291,14 @@ assert res.data[0] == 7 assert res.data[1] == 11 + def f(): + a1 = numpy.array([1,2]) + return a1 + 2 + + res = interpret(f, []) + assert res.data[0] == 3 + assert res.data[1] == 4 + def test_specialize_array_attr(self): def f(): a = numpy.array([1,2]) From arigo at codespeak.net Sat Sep 8 10:54:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 Sep 2007 10:54:29 +0200 (CEST) Subject: [pypy-svn] r46413 - pypy/dist/pypy/translator/sandbox/test Message-ID: <20070908085429.7A1B180E3@code0.codespeak.net> Author: arigo Date: Sat Sep 8 10:54:27 2007 New Revision: 46413 Added: pypy/dist/pypy/translator/sandbox/test/test_vfs.py (contents, props changed) Log: A test for vfs.py. Naughy me for writing the code before this test. Added: pypy/dist/pypy/translator/sandbox/test/test_vfs.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/sandbox/test/test_vfs.py Sat Sep 8 10:54:27 2007 @@ -0,0 +1,92 @@ +import py +import sys, stat, os +from pypy.translator.sandbox.vfs import * +from pypy.tool.udir import udir + +HASLINK = hasattr(os, 'symlink') + +def setup_module(mod): + d = udir.ensure('test_vfs', dir=1) + d.join('file1').write('somedata1') + d.join('file2').write('somelongerdata2') + os.chmod(str(d.join('file2')), stat.S_IWUSR) # unreadable + d.join('.hidden').write('secret') + d.ensure('subdir1', dir=1).join('subfile1').write('spam') + d.ensure('.subdir2', dir=1).join('subfile2').write('secret as well') + if HASLINK: + d.join('symlink1').mksymlinkto(str(d.join('subdir1'))) + d.join('symlink2').mksymlinkto('.hidden') + d.join('symlink3').mksymlinkto('BROKEN') + + +def test_dir(): + d = Dir({'foo': Dir()}) + assert d.keys() == ['foo'] + py.test.raises(OSError, d.open) + assert 0 <= d.getsize() <= sys.maxint + d1 = d.join('foo') + assert stat.S_ISDIR(d1.kind) + assert d1.keys() == [] + py.test.raises(OSError, d.join, 'bar') + st = d.stat() + assert stat.S_ISDIR(st.st_mode) + +def test_file(): + f = File('hello world') + assert stat.S_ISREG(f.kind) + py.test.raises(OSError, f.keys) + assert f.getsize() == 11 + h = f.open() + data = h.read() + assert data == 'hello world' + h.close() + st = f.stat() + assert stat.S_ISREG(st.st_mode) + assert st.st_size == 11 + +def test_realdir_realfile(): + for show_dotfiles in [False, True]: + for follow_links in [False, True]: + v_udir = RealDir(str(udir), show_dotfiles = show_dotfiles, + follow_links = follow_links) + v_test_vfs = v_udir.join('test_vfs') + names = v_test_vfs.keys() + names.sort() + assert names == (show_dotfiles * ['.hidden', '.subdir2'] + + ['file1', 'file2', 'subdir1'] + + HASLINK * ['symlink1', 'symlink2', 'symlink3']) + py.test.raises(OSError, v_test_vfs.open) + assert 0 <= v_test_vfs.getsize() <= sys.maxint + + f = v_test_vfs.join('file1') + assert f.open().read() == 'somedata1' + + f = v_test_vfs.join('file2') + assert f.getsize() == len('somelongerdata2') + py.test.raises(OSError, f.open) + + py.test.raises(OSError, v_test_vfs.join, 'does_not_exist') + py.test.raises(OSError, v_test_vfs.join, 'symlink3') + if follow_links: + d = v_test_vfs.join('symlink1') + assert stat.S_ISDIR(d.stat().st_mode) + assert d.keys() == ['subfile1'] + assert d.join('subfile1').open().read() == 'spam' + + f = v_test_vfs.join('symlink2') + assert stat.S_ISREG(f.stat().st_mode) + assert f.open().read() == 'secret' + else: + py.test.raises(OSError, v_test_vfs.join, 'symlink1') + py.test.raises(OSError, v_test_vfs.join, 'symlink2') + + if show_dotfiles: + f = v_test_vfs.join('.hidden') + assert f.open().read() == 'secret' + + d = v_test_vfs.join('.subdir2') + assert d.keys() == ['subfile2'] + assert d.join('subfile2').open().read() == 'secret as well' + else: + py.test.raises(OSError, v_test_vfs.join, '.hidden') + py.test.raises(OSError, v_test_vfs.join, '.subdir2') From arigo at codespeak.net Sat Sep 8 10:55:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 Sep 2007 10:55:00 +0200 (CEST) Subject: [pypy-svn] r46414 - pypy/dist/pypy/translator/sandbox Message-ID: <20070908085500.EF79480E4@code0.codespeak.net> Author: arigo Date: Sat Sep 8 10:55:00 2007 New Revision: 46414 Modified: pypy/dist/pypy/translator/sandbox/vfs.py Log: Oups, this goes with the previous check-in. Modified: pypy/dist/pypy/translator/sandbox/vfs.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/vfs.py (original) +++ pypy/dist/pypy/translator/sandbox/vfs.py Sat Sep 8 10:55:00 2007 @@ -95,6 +95,13 @@ class File(FSObject): kind = stat.S_IFREG + def __init__(self, data=''): + self.data = data + def getsize(self): + return len(self.data) + def open(self): + import cStringIO + return cStringIO.StringIO(self.data) class RealFile(File): def __init__(self, path): From arigo at codespeak.net Sat Sep 8 10:55:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 Sep 2007 10:55:34 +0200 (CEST) Subject: [pypy-svn] r46415 - pypy/dist/pypy/translator/sandbox Message-ID: <20070908085534.5FEA780E3@code0.codespeak.net> Author: arigo Date: Sat Sep 8 10:55:33 2007 New Revision: 46415 Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py pypy/dist/pypy/translator/sandbox/sandlib.py Log: Move the construction of the virtual file system to a method; easier to tweak. Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Sat Sep 8 10:55:33 2007 @@ -28,20 +28,25 @@ virtual_console_isatty = True def __init__(self, executable, arguments, tmpdir): + self.executable = executable = os.path.abspath(executable) + self.tmpdir = tmpdir + super(PyPySandboxedProc, self).__init__([self.argv0] + arguments, + executable=executable) + + def build_virtual_root(self): # build a virtual file system: # * can access its own executable # * can access the pure Python libraries # * can access the temporary usession directory as /tmp - executable = os.path.abspath(executable) - if tmpdir is None: + if self.tmpdir is None: tmpdirnode = Dir({}) else: - tmpdirnode = RealDir(tmpdir) + tmpdirnode = RealDir(self.tmpdir) pypydist = os.path.dirname(os.path.abspath(autopath.pypydir)) - virtual_root = Dir({ + return Dir({ 'bin': Dir({ - 'pypy-c': RealFile(executable), + 'pypy-c': RealFile(self.executable), 'lib-python': RealDir(os.path.join(pypydist, 'lib-python')), 'pypy': Dir({ 'lib': RealDir(os.path.join(pypydist, 'pypy', 'lib')), @@ -50,10 +55,6 @@ 'tmp': tmpdirnode, }) - super(PyPySandboxedProc, self).__init__(virtual_root, - [self.argv0] + arguments, - executable=executable) - if __name__ == '__main__': from getopt import getopt # and not gnu_getopt! Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Sat Sep 8 10:55:33 2007 @@ -378,11 +378,14 @@ virtual_console_isatty = False virtual_fd_range = range(3, 50) - def __init__(self, virtual_root, *args, **kwds): + def __init__(self, *args, **kwds): super(VirtualizedSandboxedProc, self).__init__(*args, **kwds) - self.virtual_root = virtual_root + self.virtual_root = self.build_virtual_root() self.open_fds = {} # {virtual_fd: real_file_object} + def build_virtual_root(self): + raise NotImplementedError("must be overridden") + def do_ll_os__ll_os_envitems(self): return self.virtual_env.items() From arigo at codespeak.net Sat Sep 8 11:03:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 Sep 2007 11:03:28 +0200 (CEST) Subject: [pypy-svn] r46416 - pypy/dist/pypy/translator/sandbox Message-ID: <20070908090328.7FC5080E3@code0.codespeak.net> Author: arigo Date: Sat Sep 8 11:03:28 2007 New Revision: 46416 Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py Log: Document how to use ledit with pypy_interact. Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Sat Sep 8 11:03:28 2007 @@ -12,6 +12,11 @@ with the 'k', 'm' or 'g' suffix respectively. ATM this only works with PyPy translated with Boehm. --timeout=N limit execution time to N (real-time) seconds. + +Note that you can get readline-like behavior with a tool like 'ledit', +provided you use enough -u options: + + ledit python -u pypy_interact.py pypy-c-sandbox -u """ import sys, os From arigo at codespeak.net Sat Sep 8 11:30:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 Sep 2007 11:30:32 +0200 (CEST) Subject: [pypy-svn] r46417 - in pypy/dist/pypy/translator/sandbox: . test Message-ID: <20070908093032.D857880D6@code0.codespeak.net> Author: arigo Date: Sat Sep 8 11:30:32 2007 New Revision: 46417 Added: pypy/dist/pypy/translator/sandbox/test/test_pypy_interact.py (contents, props changed) Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py pypy/dist/pypy/translator/sandbox/sandlib.py Log: A test for pypy_interact.py, emulating the I/O calls pypy does at start-up. Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Sat Sep 8 11:30:32 2007 @@ -32,7 +32,7 @@ virtual_env = {} virtual_console_isatty = True - def __init__(self, executable, arguments, tmpdir): + def __init__(self, executable, arguments, tmpdir=None): self.executable = executable = os.path.abspath(executable) self.tmpdir = tmpdir super(PyPySandboxedProc, self).__init__([self.argv0] + arguments, Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Sat Sep 8 11:30:32 2007 @@ -314,6 +314,7 @@ self._input = None self._output = None self._error = None + return returncode def do_ll_os__ll_os_read(self, fd, size): if fd == 0: Added: pypy/dist/pypy/translator/sandbox/test/test_pypy_interact.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/sandbox/test/test_pypy_interact.py Sat Sep 8 11:30:32 2007 @@ -0,0 +1,75 @@ +import autopath +import os, sys, stat, errno +from pypy.translator.sandbox.pypy_interact import PyPySandboxedProc +from pypy.translator.interactive import Translation + +SITE_PY_CONTENT = open(os.path.join(autopath.pypydir, os.pardir, + 'lib-python', 'modified-2.4.1', 'site.py'), + 'rb').read() +ERROR_TEXT = os.strerror(errno.ENOENT) + +def assert_(cond, text): + if not cond: + print "assert failed:", text + raise AssertionError + +def mini_pypy_like_entry_point(argv): + """An RPython standalone executable that does the same kind of I/O as + PyPy when it starts up. + """ + assert_(len(argv) == 3, "expected len(argv) == 3") + assert_(argv[1] == 'foo', "bad argv[1]") + assert_(argv[2] == 'bar', "bad argv[2]") + env = os.environ.items() + assert_(len(env) == 0, "empty environment expected") + assert_(argv[0] == '/bin/pypy-c', "bad argv[0]") + st = os.lstat('/bin/pypy-c') + assert_(stat.S_ISREG(st.st_mode), "bad st_mode for /bin/pypy-c") + for dirname in ['/bin/lib-python/2.4.1', '/bin/pypy/lib']: + st = os.stat(dirname) + assert_(stat.S_ISDIR(st.st_mode), "bad st_mode for " + dirname) + assert_(os.environ.get('PYTHONPATH') is None, "unexpected $PYTHONPATH") + try: + os.stat('site') + except OSError: + pass + else: + assert_(False, "os.stat('site') should have failed") + st = os.stat('/bin/lib-python/modified-2.4.1/site.py') + assert_(stat.S_ISREG(st.st_mode), "bad st_mode for .../site.py") + fd = os.open('/bin/lib-python/modified-2.4.1/site.py', os.O_RDONLY, 0666) + length = 8192 + ofs = 0 + while True: + data = os.read(fd, length) + if not data: break + end = ofs+length + if end > len(SITE_PY_CONTENT): + end = len(SITE_PY_CONTENT) + assert_(data == SITE_PY_CONTENT[ofs:end], "bad data from site.py") + ofs = end + os.close(fd) + assert_(ofs == len(SITE_PY_CONTENT), "not enough data from site.py") + assert_(os.getcwd() == '/tmp', "bad cwd") + assert_(os.strerror(errno.ENOENT) == ERROR_TEXT, "bad strerror(ENOENT)") + assert_(os.isatty(0), "isatty(0) returned False") + # an obvious 'attack' + try: + os.open('/spam', os.O_RDWR | os.O_CREAT, 0666) + except OSError: + pass + else: + assert_(False, "os.open('/spam') should have failed") + return 0 + + +def setup_module(mod): + t = Translation(mini_pypy_like_entry_point, backend='c', + standalone=True, sandbox=True) + mod.executable = t.compile() + + +def test_run(): + sandproc = PyPySandboxedProc(executable, ['foo', 'bar']) + returncode = sandproc.interact() + assert returncode == 0 From arigo at codespeak.net Sat Sep 8 12:09:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 Sep 2007 12:09:12 +0200 (CEST) Subject: [pypy-svn] r46418 - pypy/dist/pypy/translator/sandbox Message-ID: <20070908100912.ADDDA80D6@code0.codespeak.net> Author: arigo Date: Sat Sep 8 12:09:11 2007 New Revision: 46418 Modified: pypy/dist/pypy/translator/sandbox/sandlib.py Log: Fix --timeout in the presence of ledit. Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Sat Sep 8 12:09:11 2007 @@ -320,7 +320,8 @@ if fd == 0: if self._input is None: return "" - elif self._input.isatty(): + elif (getattr(self, 'virtual_console_isatty', False) or + self._input.isatty()): # don't wait for all 'size' chars if reading from a tty, # to avoid blocking. Instead, stop after reading a line. From arigo at codespeak.net Sat Sep 8 12:20:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 Sep 2007 12:20:24 +0200 (CEST) Subject: [pypy-svn] r46419 - pypy/dist/pypy/translator/sandbox Message-ID: <20070908102024.21EC080D8@code0.codespeak.net> Author: arigo Date: Sat Sep 8 12:20:23 2007 New Revision: 46419 Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py pypy/dist/pypy/translator/sandbox/sandlib.py Log: An option to log all interactive input sent to the sandboxed process. Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Sat Sep 8 12:20:23 2007 @@ -12,6 +12,7 @@ with the 'k', 'm' or 'g' suffix respectively. ATM this only works with PyPy translated with Boehm. --timeout=N limit execution time to N (real-time) seconds. + --log=FILE log all user input into the FILE Note that you can get readline-like behavior with a tool like 'ledit', provided you use enough -u options: @@ -64,9 +65,11 @@ if __name__ == '__main__': from getopt import getopt # and not gnu_getopt! options, arguments = getopt(sys.argv[1:], 't:h', - ['tmp=', 'heapsize=', 'timeout=', 'help']) + ['tmp=', 'heapsize=', 'timeout=', 'log=', + 'help']) tmpdir = None timeout = None + logfile = None extraoptions = [] def help(): @@ -96,6 +99,8 @@ extraoptions[:0] = ['--heapsize', str(bytes)] elif option == '--timeout': timeout = int(value) + elif option == '--log': + logfile = value elif option in ['-h', '--help']: help() else: @@ -108,6 +113,8 @@ tmpdir=tmpdir) if timeout is not None: sandproc.settimeout(timeout, interrupt_main=True) + if logfile is not None: + sandproc.setlogfile(logfile) try: sandproc.interact() finally: Modified: pypy/dist/pypy/translator/sandbox/sandlib.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/sandlib.py (original) +++ pypy/dist/pypy/translator/sandbox/sandlib.py Sat Sep 8 12:20:23 2007 @@ -277,6 +277,7 @@ _input = None _output = None _error = None + inputlogfile = None def communicate(self, input=None): """Send data to stdin. Read data from stdout and stderr, @@ -316,6 +317,9 @@ self._error = None return returncode + def setlogfile(self, filename): + self.inputlogfile = open(filename, 'a') + def do_ll_os__ll_os_read(self, fd, size): if fd == 0: if self._input is None: @@ -329,11 +333,14 @@ # only time that counts as idle. self.enter_idle() try: - return self._input.readline(size) + inputdata = self._input.readline(size) finally: self.leave_idle() else: - return self._input.read(size) + inputdata = self._input.read(size) + if self.inputlogfile is not None: + self.inputlogfile.write(inputdata) + return inputdata raise OSError("trying to read from fd %d" % (fd,)) def do_ll_os__ll_os_write(self, fd, data): From mwh at codespeak.net Sat Sep 8 15:04:24 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 8 Sep 2007 15:04:24 +0200 (CEST) Subject: [pypy-svn] r46420 - pypy/extradoc/talk/pyconuk07/pyconuk07.key Message-ID: <20070908130424.E831080D7@code0.codespeak.net> Author: mwh Date: Sat Sep 8 15:04:23 2007 New Revision: 46420 Removed: pypy/extradoc/talk/pyconuk07/pyconuk07.key/ Log: remove pyconuk07.key as keynote is extremely svn hostile From mwh at codespeak.net Sat Sep 8 15:06:27 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 8 Sep 2007 15:06:27 +0200 (CEST) Subject: [pypy-svn] r46421 - pypy/extradoc/talk/pyconuk07 Message-ID: <20070908130627.7C63880E7@code0.codespeak.net> Author: mwh Date: Sat Sep 8 15:06:27 2007 New Revision: 46421 Added: pypy/extradoc/talk/pyconuk07/pyconuk07.dmg (contents, props changed) Modified: pypy/extradoc/talk/pyconuk07/pyconuk07.pdf Log: add the talk as given, as pdf and dmg-packed keynote file Added: pypy/extradoc/talk/pyconuk07/pyconuk07.dmg ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/pyconuk07/pyconuk07.pdf ============================================================================== Binary files. No diff available. From simonb at codespeak.net Sat Sep 8 23:20:37 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Sat, 8 Sep 2007 23:20:37 +0200 (CEST) Subject: [pypy-svn] r46423 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070908212037.BAC9B80F0@code0.codespeak.net> Author: simonb Date: Sat Sep 8 23:20:36 2007 New Revision: 46423 Modified: pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: basic numpy arithmetic working. Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Sat Sep 8 23:20:36 2007 @@ -195,15 +195,6 @@ it.dataptr[0] = value iter_next(it) -def ll_array_add(it0, it1, it2, iter_next): - debug_assert(it0.size == it1.size, "it0.size == it1.size") - debug_assert(it1.size == it2.size, "it0.size == it1.size") - while it0.index < it0.size: - it0.dataptr[0] = it1.dataptr[0] + it2.dataptr[0] - iter_next(it0) - iter_next(it1) - iter_next(it2) - def dim_of_ITER(ITER): return ITER.TO.coordinates.length @@ -310,41 +301,57 @@ key = self.__class__, self.typecode, self.ndim return key +def ll_array_binop(it0, it1, it2, iter_next, binop): + debug_assert(it0.size == it1.size, "it0.size == it1.size") + debug_assert(it1.size == it2.size, "it0.size == it1.size") + while it0.index < it0.size: + it0.dataptr[0] = binop(it1.dataptr[0], it2.dataptr[0]) + iter_next(it0) + iter_next(it1) + iter_next(it2) -class __extend__(pairtype(ArrayRepr, ArrayRepr)): - def rtype_add((r_array1, r_array2), hop): - v_array1, v_array2 = hop.inputargs(r_array1, r_array2) - r_array0 = hop.r_result - cARRAY = hop.inputconst(Void, r_array0.ARRAY.TO) - # We build a contiguous "result" array - # from the largest of the two args: - v_bigarray = hop.gendirectcall(ll_find_largest, cARRAY, v_array1, v_array2) - v_array0 = hop.gendirectcall(ll_build_like, cARRAY, v_bigarray) - iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(r_array0.ndim) - cITER = hop.inputconst(Void, r_array0.ITER.TO) - creset = hop.inputconst(Void, iter_reset) - cbroadcast = hop.inputconst(Void, iter_broadcast) - cnext = hop.inputconst(Void, iter_next) - v_it0 = hop.gendirectcall(iter_new, cITER, v_array0, v_bigarray, creset, cbroadcast) - v_it1 = hop.gendirectcall(iter_new, cITER, v_array1, v_bigarray, creset, cbroadcast) - v_it2 = hop.gendirectcall(iter_new, cITER, v_array2, v_bigarray, creset, cbroadcast) - return hop.gendirectcall(ll_array_add, v_it0, v_it1, v_it2, cnext) - -class __extend__(pairtype(ArrayRepr, Repr)): - def rtype_add((r_array, r_ob), hop): - assert 0 - v_ob = hop.inputarg(r_ob, 1) - if isinstance(r_ob.lowleveltype, Primitive): - r_item, v_item = convert_scalar_to_array(r_array, v_ob, hop.llops) - -class __extend__(pairtype(Repr, ArrayRepr)): - def rtype_add((r_ob, r_array), hop): - # XX ach! how to get this to work ?? - assert 0 - return pair(r_array, r_ob).rtype_add(hop) - +def _rtype_binop(r_array0, r_array1, r_array2, v_array1, v_array2, hop, binop): + cARRAY = hop.inputconst(Void, r_array0.ARRAY.TO) + # We build a contiguous "result" array + # from the largest of the two args: + v_array0 = hop.gendirectcall(ll_build_like, cARRAY, v_array1, v_array2) + iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(r_array0.ndim) + cITER = hop.inputconst(Void, r_array0.ITER.TO) + creset = hop.inputconst(Void, iter_reset) + cbroadcast = hop.inputconst(Void, iter_broadcast) + cnext = hop.inputconst(Void, iter_next) + v_it0 = hop.gendirectcall(iter_new, cITER, v_array0, v_array0, creset, cbroadcast) + v_it1 = hop.gendirectcall(iter_new, cITER, v_array1, v_array0, creset, cbroadcast) + v_it2 = hop.gendirectcall(iter_new, cITER, v_array2, v_array0, creset, cbroadcast) + cbinop = hop.inputconst(Void, binop) + hop.gendirectcall(ll_array_binop, v_it0, v_it1, v_it2, cnext, cbinop) + return v_array0 + +def rtype_binop((r_array1, r_array2), hop, binop): + r_array0 = hop.r_result + v_array1, v_array2 = hop.inputargs(r_array1, r_array2) + + if isinstance(r_array1.lowleveltype, Primitive): + r_array1, v_array1 = convert_scalar_to_array(r_array0, v_array1, hop.llops) + elif not isinstance(r_array1, ArrayRepr): + raise TyperError("can't operate with %s"%r_array1) + + if isinstance(r_array2.lowleveltype, Primitive): + r_array2, v_array2 = convert_scalar_to_array(r_array0, v_array2, hop.llops) + elif not isinstance(r_array2, ArrayRepr): + raise TyperError("can't operate with %s"%r_array2) + print "rtype_binop", binop, binop(2,3) + return _rtype_binop(r_array0, r_array1, r_array2, v_array1, v_array2, hop, binop) + +for tp in (pairtype(ArrayRepr, ArrayRepr), + pairtype(ArrayRepr, Repr), + pairtype(Repr, ArrayRepr)): + for (binop, methname) in ((lambda a,b:a+b, "rtype_add"), + (lambda a,b:a-b, "rtype_sub"), + (lambda a,b:a*b, "rtype_mul"), + (lambda a,b:a/b, "rtype_div")): + setattr(tp, methname, lambda self, hop, binop=binop : rtype_binop(self, hop, binop)) - def gen_getset_item(ndim): unrolling_dims = unrolling_iterable(range(ndim)) def ll_get_item(ARRAY, ao, tpl): @@ -447,7 +454,6 @@ s_array = r_array.s_array.get_one_dim() r_array = llops.rtyper.getrepr(s_array) from pypy.rpython.rmodel import inputconst -# ARRAY, _ = ArrayRepr.make_types(1, ITEM) cARRAY = inputconst(Void, r_array.ARRAY.TO) cITEM = inputconst(Void, r_array.ITEM) v_casted = llops.genop("cast_primitive", [v_item], r_array.ITEM) @@ -602,47 +608,47 @@ array.dataptr = ao.dataptr return array -def ll_find_largest(ARRAY, array0, array1): +def ll_build_like(ARRAY, array0, array1): + # Build with shape from the largest of array0 or array1. + # Note we cannot take the union of array0 and array1. + ndim = max(array0.ndim, array1.ndim) + array = ll_allocate(ARRAY, ndim) sz0 = ll_mul_list(array0.shape, array0.ndim) sz1 = ll_mul_list(array1.shape, array1.ndim) - # XXX - if sz0 > sz1: - return array0 - return array1 - -def ll_build_like(ARRAY, ao): - array = ll_allocate(ARRAY, ao.ndim) - sz = ll_mul_list(ao.shape) + sz = max(sz0, sz1) array.data = malloc(ARRAY.data.TO, sz) - array.dataptr = array.data + array.dataptr = direct_arrayitems(array.data) itemsize = 1 - i = ao.ndim - 1 + i = ndim - 1 while i >= 0: - size = ao.shape[i] + if sz0>sz1: + size = array0.shape[i] + else: + size = array1.shape[i] array.shape[i] = size array.strides[i] = itemsize itemsize *= size i -= 1 return array -def ll_setitem1(array, index, item): - array.data[index] = item - -def ll_getitem1(array, index): - return array.data[index] - -def ll_add(ARRAY, a1, a2): - size = a1.shape[0] - if size != a2.shape[0]: - raise ValueError - array = malloc(ARRAY) - array.data = malloc(ARRAY.data.TO, size) - i = 0 - while i < size: - array.data[i] = a1.data[i] + a2.data[i] - i += 1 - array.dataptr = direct_arrayitems(array.data) - return array +#def ll_setitem1(array, index, item): +# array.data[index] = item +# +#def ll_getitem1(array, index): +# return array.data[index] + +#def ll_add(ARRAY, a1, a2): +# size = a1.shape[0] +# if size != a2.shape[0]: +# raise ValueError +# array = malloc(ARRAY) +# array.data = malloc(ARRAY.data.TO, size) +# i = 0 +# while i < size: +# array.data[i] = a1.data[i] + a2.data[i] +# i += 1 +# array.dataptr = direct_arrayitems(array.data) +# return array def ll_transpose(ARRAY, ao): ndim = ao.ndim Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Sat Sep 8 23:20:36 2007 @@ -106,7 +106,6 @@ assert s.ndim == 2 def test_annotate_array_add(self): - py.test.skip("broken ATM") def f(): a1 = numpy.array([1,2]) a2 = numpy.array([6,9]) @@ -118,7 +117,6 @@ assert s.typecode == 'i' def test_annotate_array_add_coerce(self): - py.test.skip("broken ATM") def f(): a1 = numpy.array([1,2]) a2 = numpy.array([6.,9.]) @@ -130,7 +128,6 @@ assert s.typecode == 'd' def test_annotate_array_add_2(self): - py.test.skip("broken ATM") def f(): a = numpy.array([1,2]) a = a + 3 @@ -280,25 +277,6 @@ res = interpret(access_with_variable, []) assert res == 45 - def test_specialize_array_add(self): - py.test.skip("broken ATM") - def f(): - a1 = numpy.array([1.,2.]) - a2 = numpy.array([6,9]) - return a1 + a2 - - res = interpret(f, []) - assert res.data[0] == 7 - assert res.data[1] == 11 - - def f(): - a1 = numpy.array([1,2]) - return a1 + 2 - - res = interpret(f, []) - assert res.data[0] == 3 - assert res.data[1] == 4 - def test_specialize_array_attr(self): def f(): a = numpy.array([1,2]) @@ -388,6 +366,29 @@ for i in range(len(data)): assert res.dataptr[i] == data[i] + def test_specialize_list_coerce(self): + py.test.skip('coercion not implemented') + def f(): + a = numpy.zeros((3,4), dtype='i') + a[:] = [3.,4.,4.,2.] + return a + res = interpret(f, []) + data = [3,4,4,2]*3 + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def test_specialize_rhs_coerce(self): + py.test.skip('coercion not implemented') + def f(): + a = numpy.zeros((4,), dtype='i') + b = numpy.array([3.,4.,4.,2.]) + a[:] = b + return a + res = interpret(f, []) + data = [3,4,4,2] + for i in range(len(data)): + assert res.dataptr[i] == data[i] + def test_specialize_slice_1_0(self): def f(): a = numpy.zeros((12,), dtype='i') @@ -518,7 +519,101 @@ from pypy.translator.backendopt.test.test_malloc import TestLLTypeMallocRemoval TestLLTypeMallocRemoval.check_malloc_removed(graph) + def test_specialize_array_add_1_0(self): + def f(): + a1 = numpy.array(range(4,10)) + a2 = numpy.array([3]) + return a1 + a2 + data = [i+3 for i in range(4,10)] + res = interpret(f, []) + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def f(): + a = numpy.array(range(4,10)) + return a + 3 + res = interpret(f, []) + for i in range(len(data)): + assert res.dataptr[i] == data[i] + def f(): + a = numpy.array(range(4,10)) + return 3 + a + res = interpret(f, []) + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def test_specialize_array_sub_1_0(self): + def f(): + a1 = numpy.array(range(4,10)) + a2 = numpy.array([3]) + return a1 - a2 + data = [i-3 for i in range(4,10)] + res = interpret(f, []) + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def f(): + a = numpy.array(range(4,10)) + return a - 3 + res = interpret(f, []) + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def f(): + a = numpy.array(range(4,10)) + return 3 - a + data = [3-i for i in range(4,10)] + res = interpret(f, []) + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + + def test_specialize_array_add_1_1(self): + def f(): + a1 = numpy.array([1,2]) + a2 = numpy.array([6,9]) + return a1 + a2 + + res = interpret(f, []) + assert res.data[0] == 7 + assert res.data[1] == 11 + + def test_specialize_array_add_1_1_coerce(self): + py.test.skip('coercion not implemented') + def f(): + a1 = numpy.array([1,2]) + a2 = numpy.array([6.,9.]) + return a1 + a2 + + res = interpret(f, []) + assert res.data[0] == 7. + assert res.data[1] == 11. + + def test_specialize_array_add_2_1(self): + def f(): + a1 = numpy.array([1,2,3,4]).reshape((2,2)) + a2 = numpy.array([5,6]) + return a1 + a2 + + res = interpret(f, []) + data = [6,8,8,10] + for i in range(len(data)): + assert res.dataptr[i] == data[i] + assert res.shape[0] == 2 + assert res.shape[1] == 2 + assert res.strides[0] == 2 + assert res.strides[1] == 1 + + def test_specialize_array_mul_0_2(self): + def f(): + a = numpy.array([1,2,3,4]).reshape((2,2)) + return 5*a + + res = interpret(f, []) + data = [1,2,3,4] + for i in range(len(data)): + assert res.dataptr[i] == 5*data[i] class Test_compile: def setup_class(self): From tismer at codespeak.net Sun Sep 9 16:45:47 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 9 Sep 2007 16:45:47 +0200 (CEST) Subject: [pypy-svn] r46424 - in pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key: . Contents QuickLook thumbs Message-ID: <20070909144547.C2B0C80C4@code0.codespeak.net> Author: tismer Date: Sun Sep 9 16:45:45 2007 New Revision: 46424 Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/ pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/Contents/ pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/Contents/PkgInfo pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/QuickLook/ pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/QuickLook/Thumbnail.jpg (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/index.apxl.gz (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/kaa.jpg (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/pickledtasklet.jpg (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/py-web1.png (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/ pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/mt0-0.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st0.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st10.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st11.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st12.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st13.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st14.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st15.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st16.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st17.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st18.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st19.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st2.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st20.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-1.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-2.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-3.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-4.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st22.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st23.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st24.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st3.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st4.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st5.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st6.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st7.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st8.tiff (contents, props changed) pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st9.tiff (contents, props changed) Log: stackless pypy talk. hey, this time people seemed to be happy Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/Contents/PkgInfo ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/Contents/PkgInfo Sun Sep 9 16:45:45 2007 @@ -0,0 +1 @@ +???????? \ No newline at end of file Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/QuickLook/Thumbnail.jpg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/index.apxl.gz ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/kaa.jpg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/pickledtasklet.jpg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/py-web1.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/mt0-0.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st0.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st10.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st11.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st12.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st13.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st14.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st15.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st16.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st17.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st18.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st19.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st2.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st20.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-1.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-2.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-3.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21-4.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st21.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st22.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st23.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st24.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st3.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st4.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st5.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st6.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st7.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st8.tiff ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/thumbs/st9.tiff ============================================================================== Binary file. No diff available. From tismer at codespeak.net Sun Sep 9 16:46:10 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 9 Sep 2007 16:46:10 +0200 (CEST) Subject: [pypy-svn] r46425 - pypy/extradoc/talk/pyconuk07 Message-ID: <20070909144610.555D380C2@code0.codespeak.net> Author: tismer Date: Sun Sep 9 16:46:09 2007 New Revision: 46425 Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.pdf (contents, props changed) Log: stackless pypy talk. hey, this time people seemed to be happy Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.pdf ============================================================================== Binary file. No diff available. From niko at codespeak.net Sun Sep 9 22:04:22 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Sun, 9 Sep 2007 22:04:22 +0200 (CEST) Subject: [pypy-svn] r46428 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20070909200422.8AFDD80BE@code0.codespeak.net> Author: niko Date: Sun Sep 9 22:04:20 2007 New Revision: 46428 Added: pypy/dist/pypy/translator/jvm/src/pypy/Constants.java pypy/dist/pypy/translator/jvm/src/pypy/StatResult.java Modified: pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/prebuiltnodes.py pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/test/test_builtin.py pypy/dist/pypy/translator/jvm/typesystem.py Log: make os.stat work, and correct the implementation of getcwdf Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Sun Sep 9 22:04:20 2007 @@ -6,6 +6,7 @@ from cStringIO import StringIO from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype, rclass +from pypy.rpython.ootypesystem.module import ll_os from pypy.translator.jvm import typesystem as jvmtype from pypy.translator.jvm import node, methods from pypy.translator.jvm.option import getoption @@ -460,8 +461,8 @@ ootype.UniChar: jvmtype.jChar, ootype.Class: jvmtype.jClass, ootype.ROOT: jvmtype.jObject, # treat like a scalar - WeakGcAddress: jvmtype.jWeakRef - } + WeakGcAddress: jvmtype.jWeakRef, + } # Dictionary for non-scalar types; in this case, if we see the key, we # will return a JvmBuiltInType based on the value @@ -472,6 +473,7 @@ ootype.Dict: jvmtype.jHashMap, ootype.DictItemsIterator:jvmtype.jPyPyDictItemsIterator, ootype.CustomDict: jvmtype.jPyPyCustomDict, + ll_os.STAT_RESULT: jvmtype.jPyPyStatResult, } def lltype_to_cts(self, OOT): Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Sun Sep 9 22:04:20 2007 @@ -173,6 +173,8 @@ 'FilterSet', 'HashCode', 'Interlink', + 'Constants', + 'StatResult', 'PyPy', )) Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Sun Sep 9 22:04:20 2007 @@ -748,13 +748,21 @@ self.interlink = interlink self.name = name self.helper = helper - self.method_obj = jvmgen.Method.v(interlink, self.name, [], jVoid) + self.method_obj = jvmgen.Method.v(interlink, + self.name, + self.helper.argument_types, + jVoid) def method(self): return self.method_obj def render(self, gen): - gen.begin_function(self.name, (), [self.interlink], jVoid) + argtypes = [self.interlink] + list(self.helper.argument_types) + gen.begin_function(self.name, (), argtypes, jVoid) + varindex = 1 + for argty in self.helper.argument_types: + gen.load_jvm_var(argty, varindex) + varindex += argty.descriptor.type_width() gen.emit(self.helper) gen.return_val(jVoid) gen.end_function() Modified: pypy/dist/pypy/translator/jvm/prebuiltnodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/prebuiltnodes.py (original) +++ pypy/dist/pypy/translator/jvm/prebuiltnodes.py Sun Sep 9 22:04:20 2007 @@ -32,5 +32,8 @@ graph = graphof(translator, func) helpers[func.func_name] = db.pending_function(graph) + raise_OSError_graph = translator.rtyper.exceptiondata.fn_raise_OSError.graph + helpers["throwOSError"] = db.pending_function(raise_OSError_graph) + db.create_interlink_node(helpers) Added: pypy/dist/pypy/translator/jvm/src/pypy/Constants.java ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/src/pypy/Constants.java Sun Sep 9 22:04:20 2007 @@ -0,0 +1,132 @@ +package pypy; + +public interface Constants { + public static final int EPERM = 1; // Operation not permitted + public static final int ENOENT = 2; // No such file or directory + public static final int ESRCH = 3; // No such process + public static final int EINTR = 4; // Interrupted system call + public static final int EIO = 5; // I/O error + public static final int ENXIO = 6; // No such device or address + public static final int E2BIG = 7; // Arg list too long + public static final int ENOEXEC = 8; // Exec format error + public static final int EBADF = 9; // Bad file number + public static final int ECHILD = 10; // No child processes + public static final int EAGAIN = 11; // Try again + public static final int ENOMEM = 12; // Out of memory + public static final int EACCES = 13; // Permission denied + public static final int EFAULT = 14; // Bad address + public static final int ENOTBLK = 15; // Block device required + public static final int EBUSY = 16; // Device or resource busy + public static final int EEXIST = 17; // File exists + public static final int EXDEV = 18; // Cross-device link + public static final int ENODEV = 19; // No such device + public static final int ENOTDIR = 20; // Not a directory + public static final int EISDIR = 21; // Is a directory + public static final int EINVAL = 22; // Invalid argument + public static final int ENFILE = 23; // File table overflow + public static final int EMFILE = 24; // Too many open files + public static final int ENOTTY = 25; // Not a typewriter + public static final int ETXTBSY = 26; // Text file busy + public static final int EFBIG = 27; // File too large + public static final int ENOSPC = 28; // No space left on device + public static final int ESPIPE = 29; // Illegal seek + public static final int EROFS = 30; // Read-only file system + public static final int EMLINK = 31; // Too many links + public static final int EPIPE = 32; // Broken pipe + public static final int EDOM = 33; // Math argument out of domain of func + public static final int ERANGE = 34; // Math result not representable + public static final int EDEADLK = 35; // Resource deadlock would occur + public static final int ENAMETOOLONG = 36; // File name too long + public static final int ENOLCK = 37; // No record locks available + public static final int ENOSYS = 38; // Function not implemented + public static final int ENOTEMPTY = 39; // Directory not empty + public static final int ELOOP = 40; // Too many symbolic links encountered + public static final int EWOULDBLOCK = EAGAIN; // Operation would block + public static final int ENOMSG = 42; // No message of desired type + public static final int EIDRM = 43; // Identifier removed + public static final int ECHRNG = 44; // Channel number out of range + public static final int EL2NSYNC = 45; // Level 2 not synchronized + public static final int EL3HLT = 46; // Level 3 halted + public static final int EL3RST = 47; // Level 3 reset + public static final int ELNRNG = 48; // Link number out of range + public static final int EUNATCH = 49; // Protocol driver not attached + public static final int ENOCSI = 50; // No CSI structure available + public static final int EL2HLT = 51; // Level 2 halted + public static final int EBADE = 52; // Invalid exchange + public static final int EBADR = 53; // Invalid request descriptor + public static final int EXFULL = 54; // Exchange full + public static final int ENOANO = 55; // No anode + public static final int EBADRQC = 56; // Invalid request code + public static final int EBADSLT = 57; // Invalid slot + public static final int EDEADLOCK = EDEADLK; + public static final int EBFONT = 59; // Bad font file format + public static final int ENOSTR = 60; // Device not a stream + public static final int ENODATA = 61; // No data available + public static final int ETIME = 62; // Timer expired + public static final int ENOSR = 63; // Out of streams resources + public static final int ENONET = 64; // Machine is not on the network + public static final int ENOPKG = 65; // Package not installed + public static final int EREMOTE = 66; // Object is remote + public static final int ENOLINK = 67; // Link has been severed + public static final int EADV = 68; // Advertise error + public static final int ESRMNT = 69; // Srmount error + public static final int ECOMM = 70; // Communication error on send + public static final int EPROTO = 71; // Protocol error + public static final int EMULTIHOP = 72; // Multihop attempted + public static final int EDOTDOT = 73; // RFS specific error + public static final int EBADMSG = 74; // Not a data message + public static final int EOVERFLOW = 75; // Value too large for defined data type + public static final int ENOTUNIQ = 76; // Name not unique on network + public static final int EBADFD = 77; // File descriptor in bad state + public static final int EREMCHG = 78; // Remote address changed + public static final int ELIBACC = 79; // Can not access a needed shared library + public static final int ELIBBAD = 80; // Accessing a corrupted shared library + public static final int ELIBSCN = 81; // .lib section in a.out corrupted + public static final int ELIBMAX = 82; // Attempting to link in too many shared libraries + public static final int ELIBEXEC = 83; // Cannot exec a shared library directly + public static final int EILSEQ = 84; // Illegal byte sequence + public static final int ERESTART = 85; // Interrupted system call should be restarted + public static final int ESTRPIPE = 86; // Streams pipe error + public static final int EUSERS = 87; // Too many users + public static final int ENOTSOCK = 88; // Socket operation on non-socket + public static final int EDESTADDRREQ = 89; // Destination address required + public static final int EMSGSIZE = 90; // Message too long + public static final int EPROTOTYPE = 91; // Protocol wrong type for socket + public static final int ENOPROTOOPT = 92; // Protocol not available + public static final int EPROTONOSUPPORT = 93; // Protocol not supported + public static final int ESOCKTNOSUPPORT = 94; // Socket type not supported + public static final int EOPNOTSUPP = 95; // Operation not supported on transport endpoint + public static final int EPFNOSUPPORT = 96; // Protocol family not supported + public static final int EAFNOSUPPORT = 97; // Address family not supported by protocol + public static final int EADDRINUSE = 98; // Address already in use + public static final int EADDRNOTAVAIL = 99; // Cannot assign requested address + public static final int ENETDOWN = 100; // Network is down + public static final int ENETUNREACH = 101; // Network is unreachable + public static final int ENETRESET = 102; // Network dropped connection because of reset + public static final int ECONNABORTED = 103; // Software caused connection abort + public static final int ECONNRESET = 104; // Connection reset by peer + public static final int ENOBUFS = 105; // No buffer space available + public static final int EISCONN = 106; // Transport endpoint is already connected + public static final int ENOTCONN = 107; // Transport endpoint is not connected + public static final int ESHUTDOWN = 108; // Cannot send after transport endpoint shutdown + public static final int ETOOMANYREFS = 109; // Too many references: cannot splice + public static final int ETIMEDOUT = 110; // Connection timed out + public static final int ECONNREFUSED = 111; // Connection refused + public static final int EHOSTDOWN = 112; // Host is down + public static final int EHOSTUNREACH = 113; // No route to host + public static final int EALREADY = 114; // Operation already in progress + public static final int EINPROGRESS = 115; // Operation now in progress + public static final int ESTALE = 116; // Stale NFS file handle + public static final int EUCLEAN = 117; // Structure needs cleaning + public static final int ENOTNAM = 118; // Not a XENIX named type file + public static final int ENAVAIL = 119; // No XENIX semaphores available + public static final int EISNAM = 120; // Is a named type file + public static final int EREMOTEIO = 121; // Remote I/O error + public static final int EDQUOT = 122; // Quota exceeded + public static final int ENOMEDIUM = 123; // No medium found + public static final int EMEDIUMTYPE = 124; // Wrong medium type + + public static final int S_IFDIR = 16384; + public static final int S_IFREG = 32768; + +} Modified: pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java Sun Sep 9 22:04:20 2007 @@ -14,4 +14,5 @@ public void throwIndexError(); public void throwOverflowError(); public void throwValueError(); + public void throwOSError(int errCode); } Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Sun Sep 9 22:04:20 2007 @@ -1,5 +1,6 @@ package pypy; +import java.io.File; import java.util.List; import java.util.ArrayList; import java.util.HashMap; @@ -14,7 +15,7 @@ * I can't switch my mind to camelCase when working so closely with * Python mere minutes before. */ -public class PyPy { +public class PyPy implements Constants { public static Interlink interlink; @@ -809,7 +810,30 @@ public static String ll_os_getcwd() { - return "/tmp"; + return System.getProperty("user.dir"); + } + + public static StatResult ll_os_stat(String path) + { + if (path.equals("")) + throwOSError(ENOENT, "No such file or directory: ''"); + + File f = new File(path); + + if (f.exists()) { + StatResult res = new StatResult(); + if (f.isDirectory()) + res.setMode(S_IFDIR); + else { + res.setMode(S_IFREG); + res.setSize(f.length()); + res.setMtime(f.lastModified()); + } + return res; + } + + throwOSError(ENOENT, "No such file or directory: '"+path+"'"); + return null; // never reached } // ---------------------------------------------------------------------- @@ -927,6 +951,10 @@ public static void throwValueError() { interlink.throwValueError(); } + + public static void throwOSError(int errCode, String errText) { + interlink.throwOSError(errCode); // errText currently ignored... fix? + } // ---------------------------------------------------------------------- // Self Test Added: pypy/dist/pypy/translator/jvm/src/pypy/StatResult.java ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/src/pypy/StatResult.java Sun Sep 9 22:04:20 2007 @@ -0,0 +1,19 @@ +package pypy; + +/** + * Class returned by the Stat function. The Rpython type is a record + * type which is special-cased in database.py to return an instance of + * this class. The fields are named item0...itemN to conform with + * Rpython, but there are friendly accessors for humans to use. + * + *

The actual stat() function is defined in PyPy.java. + */ +class StatResult { + public int item0; + public long item6, item8; + + public void setMode(int value) { item0 = value; } + public void setSize(long value) { item6 = value; } + public void setMtime(long value) { item8 = value; } + +} \ No newline at end of file Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Sun Sep 9 22:04:20 2007 @@ -1,3 +1,4 @@ +import os, stat import py from pypy.tool import udir from pypy.translator.jvm.test.runtest import JvmTest @@ -10,9 +11,6 @@ def test_os_open(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - def test_os_getcwd(self): - py.test.skip("ll_os_getcwd is not currently implemented in the Jvm backed") - def test_os_write(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") @@ -22,12 +20,22 @@ def test_os_read(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - def test_os_path_exists(self): - py.test.skip("ll_os_stat is not currently implemented in the Jvm backed") - - def test_os_isdir(self): - py.test.skip("ll_os_stat is not currently implemented in the Jvm backed") - + def test_os_stat(self): + def fn(flag): + if flag: + return os.stat('.')[0] + else: + return os.stat('.').st_mode + mode = self.interpret(fn, [0]) + assert stat.S_ISDIR(mode) + mode = self.interpret(fn, [1]) + assert stat.S_ISDIR(mode) + + def test_os_stat_oserror(self): + def fn(): + return os.stat('/directory/unlikely/to/exists')[0] + self.interpret_raises(OSError, fn, []) + def test_builtin_math_frexp(self): py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Sun Sep 9 22:04:20 2007 @@ -177,6 +177,7 @@ jPyPyDictItemsIterator = JvmClassType('pypy.DictItemsIterator') jPyPyInterlink = JvmClassType('pypy.Interlink') jPyPyCustomDict = JvmClassType('pypy.CustomDict') +jPyPyStatResult = JvmClassType('pypy.StatResult') jArithmeticException = JvmClassType('java.lang.ArithmeticException', throwable=True) From antocuni at codespeak.net Mon Sep 10 13:09:37 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 10 Sep 2007 13:09:37 +0200 (CEST) Subject: [pypy-svn] r46433 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070910110937.9400E810F@code0.codespeak.net> Author: antocuni Date: Mon Sep 10 13:09:36 2007 New Revision: 46433 Modified: pypy/dist/pypy/translator/jvm/src/pypy/StatResult.java Log: include all needed StatResult's fields Modified: pypy/dist/pypy/translator/jvm/src/pypy/StatResult.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/StatResult.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/StatResult.java Mon Sep 10 13:09:36 2007 @@ -9,11 +9,10 @@ *

The actual stat() function is defined in PyPy.java. */ class StatResult { - public int item0; - public long item6, item8; + public int item0, item3, item4, item5, item7, item8, item9; + public long item1, item2, item6; public void setMode(int value) { item0 = value; } public void setSize(long value) { item6 = value; } - public void setMtime(long value) { item8 = value; } - -} \ No newline at end of file + public void setMtime(int value) { item8 = value; } +} From antocuni at codespeak.net Mon Sep 10 13:21:34 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 10 Sep 2007 13:21:34 +0200 (CEST) Subject: [pypy-svn] r46434 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070910112134.5F9AB8114@code0.codespeak.net> Author: antocuni Date: Mon Sep 10 13:21:33 2007 New Revision: 46434 Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Log: dummy implementation of some external functions Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Mon Sep 10 13:21:33 2007 @@ -805,7 +805,7 @@ public static ArrayList ll_os_envitems() { - return new ArrayList(); + return new ArrayList(); // XXX } public static String ll_os_getcwd() @@ -836,6 +836,35 @@ return null; // never reached } + public static int ll_os_open(String path, int flags, int mode) + { + throwOSError(ENOENT, "DUMMY: No such file or directory: '"+path+"'"); // XXX + return -1; // never reached + } + + public static StatResult ll_os_lstat(String path) + { + return ll_os_stat(path); // XXX + } + + public static String ll_os_strerror(int errno) + { + return "errno: " + errno; + } + + public static String ll_join(String a, String b) + { + return a + "/" + b; // XXX + } + + public static String ll_strtod_formatd(String format, double d) + { + // XXX: this is really a quick hack to make things work. + // it should disappear, because this function is not + // supported by ootypesystem. + return Double.toString(d); // XXX: we are ignoring "format" + } + // ---------------------------------------------------------------------- // Exceptions // From arigo at codespeak.net Mon Sep 10 13:38:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Sep 2007 13:38:29 +0200 (CEST) Subject: [pypy-svn] r46435 - in pypy/dist/pypy: module/_weakref objspace/std Message-ID: <20070910113829.D42818117@code0.codespeak.net> Author: arigo Date: Mon Sep 10 13:38:29 2007 New Revision: 46435 Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py pypy/dist/pypy/objspace/std/typeobject.py Log: Disable type.__subclasses__() in a sandbox translation, to be fully weakref-free. Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Mon Sep 10 13:38:29 2007 @@ -129,9 +129,9 @@ space.wrap(msg)) def descr__new__weakref(space, w_subtype, w_obj, w_callable=None): + check(space) lifeline = w_obj.getweakref() if lifeline is None: - check(space) lifeline = WeakrefLifeline() w_obj.setweakref(space, lifeline) return lifeline.get_weakref(space, w_subtype, w_obj, w_callable) @@ -201,9 +201,9 @@ """Create a proxy object that weakly references 'obj'. 'callback', if given, is called with the proxy as an argument when 'obj' is about to be finalized.""" + check(space) lifeline = w_obj.getweakref() if lifeline is None: - check(space) lifeline = WeakrefLifeline() w_obj.setweakref(space, lifeline) return lifeline.get_proxy(space, w_obj, w_callable) @@ -268,6 +268,7 @@ """this is a bit like the app-level weakref.ref(), but with no fancy options like supporting subclasses of _weakref.ref and callbacks.""" + check(space) lifeline = w_obj.getweakref() if lifeline is None: lifeline = WeakrefLifeline() Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Mon Sep 10 13:38:29 2007 @@ -403,6 +403,8 @@ def add_subclass(w_self, w_subclass): space = w_self.space + if space.config.translation.sandbox: + return # XXX weakrefs are disabled in a sandbox translation ATM from pypy.module._weakref.interp__weakref import basic_weakref w_newref = basic_weakref(space, w_subclass) @@ -417,6 +419,8 @@ def remove_subclass(w_self, w_subclass): space = w_self.space + if space.config.translation.sandbox: + return # XXX weakrefs are disabled in a sandbox translation ATM for i in range(len(w_self.weak_subclasses_w)): w_ref = w_self.weak_subclasses_w[i] @@ -427,6 +431,11 @@ def get_subclasses(w_self): space = w_self.space + if space.config.translation.sandbox: + msg = ("weakrefs are disabled in a sandbox translation " + "at the moment") + raise OperationError(space.w_RuntimeError, + space.wrap(msg)) subclasses_w = [] for w_ref in w_self.weak_subclasses_w: w_ob = space.call_function(w_ref) From antocuni at codespeak.net Mon Sep 10 13:56:43 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 10 Sep 2007 13:56:43 +0200 (CEST) Subject: [pypy-svn] r46436 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070910115643.B230580BB@code0.codespeak.net> Author: antocuni Date: Mon Sep 10 13:56:41 2007 New Revision: 46436 Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Log: missing cast. Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Mon Sep 10 13:56:41 2007 @@ -827,7 +827,7 @@ else { res.setMode(S_IFREG); res.setSize(f.length()); - res.setMtime(f.lastModified()); + res.setMtime((int)f.lastModified()); } return res; } From antocuni at codespeak.net Mon Sep 10 14:01:43 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 10 Sep 2007 14:01:43 +0200 (CEST) Subject: [pypy-svn] r46437 - in pypy/dist/pypy: rpython/test translator/cli/test translator/jvm/test Message-ID: <20070910120143.CDFB18115@code0.codespeak.net> Author: antocuni Date: Mon Sep 10 14:01:43 2007 New Revision: 46437 Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py pypy/dist/pypy/rpython/test/tool.py pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/cli/test/test_builtin.py pypy/dist/pypy/translator/jvm/test/runtest.py Log: skip os.path tests only on the llinterpreter, not in the backends Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Mon Sep 10 14:01:43 2007 @@ -248,7 +248,7 @@ assert count == 1 def test_os_path_exists(self): - py.test.skip("cannot call os.stat() on the llinterp yet") + self._skip_llinterpreter("os.stat()") import os def f(fn): return os.path.exists(fn) @@ -258,7 +258,7 @@ self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False def test_os_isdir(self): - py.test.skip("cannot call os.stat() on the llinterp yet") + self._skip_llinterpreter("os.stat()") import os def f(fn): return os.path.isdir(fn) @@ -357,7 +357,7 @@ assert self.class_name(res) == 'B' def test_os_path_join(self): - self._skip_oo("os path oofakeimpl") + self._skip_llinterpreter("os path oofakeimpl", skipLL=False) import os.path def fn(a, b): return os.path.join(a, b) Modified: pypy/dist/pypy/rpython/test/tool.py ============================================================================== --- pypy/dist/pypy/rpython/test/tool.py (original) +++ pypy/dist/pypy/rpython/test/tool.py Mon Sep 10 14:01:43 2007 @@ -22,10 +22,11 @@ def is_of_type(self, x, type_): return type(x) is type_ - def _skip_oo(self, reason): - if self.type_system == 'ootype': - py.test.skip("ootypesystem doesn't support %s, yet" % reason) - + def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): + if skipLL and self.type_system == 'lltype': + py.test.skip("lltypesystem doesn't support %s, yet" % reason) + if skipOO and self.type_system == 'ootype': + py.test.skip("ootypesystem doesn't support %s, yet" % reason) class LLRtypeMixin(object): type_system = 'lltype' Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Mon Sep 10 14:01:43 2007 @@ -267,6 +267,9 @@ if platform.processor() == 'powerpc': py.test.skip('PowerPC --> %s' % reason) + def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): + pass + def interpret(self, fn, args, annotation=None, backendopt=True, exctrans=False): f = self._compile(fn, args, annotation, backendopt=backendopt, exctrans=exctrans) res = f(*args) Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/cli/test/test_builtin.py Mon Sep 10 14:01:43 2007 @@ -138,12 +138,6 @@ # XXX: remember to test ll_os_readlink and ll_os_pipe as soon as # they are implemented - def test_os_path_join(self): - import os.path - def fn(a, b): - return os.path.join(a, b) - res = self.ll_to_string(self.interpret(fn, ['a', 'b'])) - assert res == os.path.join('a', 'b') class TestCliTime(CliTest, BaseTestTime): Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Mon Sep 10 14:01:43 2007 @@ -104,6 +104,9 @@ if platform.processor() == 'powerpc': py.test.skip('PowerPC --> %s' % reason) + def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): + pass + def interpret(self, fn, args, annotation=None): detect_missing_support_programs() try: From arigo at codespeak.net Mon Sep 10 18:40:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Sep 2007 18:40:32 +0200 (CEST) Subject: [pypy-svn] r46443 - in pypy/dist/pypy: interpreter module/_weakref Message-ID: <20070910164032.3A90680EA@code0.codespeak.net> Author: arigo Date: Mon Sep 10 18:40:31 2007 New Revision: 46443 Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/mixedmodule.py pypy/dist/pypy/module/_weakref/__init__.py pypy/dist/pypy/module/_weakref/interp__weakref.py Log: Annotation fixes for --sandbox. In this version the _weakref module doesn't exist at all on sandboxed translations. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Mon Sep 10 18:40:31 2007 @@ -246,6 +246,7 @@ def setbuiltinmodule(self, importname): """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules""" import sys + from pypy.interpreter.mixedmodule import SkipModule fullname = "pypy.module.%s" % importname @@ -257,7 +258,10 @@ name = importname w_name = self.wrap(name) - w_mod = self.wrap(Module(self, w_name)) + try: + w_mod = self.wrap(Module(self, w_name)) + except SkipModule: + return None w_modules = self.sys.get('modules') self.setitem(w_modules, w_name, w_mod) return name Modified: pypy/dist/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/dist/pypy/interpreter/mixedmodule.py (original) +++ pypy/dist/pypy/interpreter/mixedmodule.py Mon Sep 10 18:40:31 2007 @@ -7,6 +7,9 @@ import inspect +class SkipModule(Exception): + """Skip the installation of this built-in module.""" + class MixedModule(Module): NOT_RPYTHON_ATTRIBUTES = ['loaders'] Modified: pypy/dist/pypy/module/_weakref/__init__.py ============================================================================== --- pypy/dist/pypy/module/_weakref/__init__.py (original) +++ pypy/dist/pypy/module/_weakref/__init__.py Mon Sep 10 18:40:31 2007 @@ -1,4 +1,4 @@ -from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.mixedmodule import MixedModule, SkipModule class Module(MixedModule): appleveldefs = { @@ -12,3 +12,9 @@ 'CallableProxyType': 'interp__weakref.W_CallableProxy', 'proxy': 'interp__weakref.proxy' } + + def __init__(self, space, *args): + if space.config.translation.sandbox: + raise SkipModule( + "XXX weakrefs are disabled in a sandbox translation ATM") + MixedModule.__init__(self, space, *args) Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Mon Sep 10 18:40:31 2007 @@ -122,14 +122,7 @@ self.w_hash = self.space.hash(w_obj) return self.w_hash -def check(space): - if space.config.translation.sandbox: - msg = "weakrefs are disabled in a sandbox translation at the moment" - raise OperationError(space.w_RuntimeError, - space.wrap(msg)) - def descr__new__weakref(space, w_subtype, w_obj, w_callable=None): - check(space) lifeline = w_obj.getweakref() if lifeline is None: lifeline = WeakrefLifeline() @@ -201,7 +194,6 @@ """Create a proxy object that weakly references 'obj'. 'callback', if given, is called with the proxy as an argument when 'obj' is about to be finalized.""" - check(space) lifeline = w_obj.getweakref() if lifeline is None: lifeline = WeakrefLifeline() @@ -268,7 +260,6 @@ """this is a bit like the app-level weakref.ref(), but with no fancy options like supporting subclasses of _weakref.ref and callbacks.""" - check(space) lifeline = w_obj.getweakref() if lifeline is None: lifeline = WeakrefLifeline() From arigo at codespeak.net Mon Sep 10 19:39:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Sep 2007 19:39:29 +0200 (CEST) Subject: [pypy-svn] r46447 - in pypy/dist/pypy/annotation: . test Message-ID: <20070910173929.7DACC80F8@code0.codespeak.net> Author: arigo Date: Mon Sep 10 19:39:28 2007 New Revision: 46447 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/test/test_annrpython.py pypy/dist/pypy/annotation/unaryop.py Log: Start of an attempt to support simple weakrefs directly in RPython. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Mon Sep 10 19:39:28 2007 @@ -11,7 +11,7 @@ from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator from pypy.annotation.model import SomePBC, SomeSlice, SomeFloat, s_None -from pypy.annotation.model import SomeExternalObject +from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeWeakGcAddress from pypy.annotation.model import SomeCTypesObject @@ -867,6 +867,16 @@ #_________________________________________ +# weakrefs + +class __extend__(pairtype(SomeWeakRef, SomeWeakRef)): + def union((s_wrf1, s_wrf2)): + basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) + if basedef is None: + return SomeObject() + return SomeWeakRef(basedef) + +#_________________________________________ # memory addresses class __extend__(pairtype(SomeAddress, SomeAddress)): Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Mon Sep 10 19:39:28 2007 @@ -8,7 +8,7 @@ from pypy.annotation.model import SomeUnicodeCodePoint, SomeAddress from pypy.annotation.model import SomeFloat, SomeWeakGcAddress, unionof from pypy.annotation.model import SomePBC, SomeInstance, SomeDict -from pypy.annotation.model import SomeExternalObject +from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata from pypy.annotation.model import s_ImpossibleValue @@ -569,6 +569,18 @@ BUILTIN_ANALYZERS[ootype.ooidentityhash] = ooidentityhash #________________________________ +# weakrefs + +import weakref + +def weakref_ref(s_obj): + if not isinstance(s_obj, SomeInstance): + raise Exception("cannot take a weakref to %r" % (s_obj,)) + return SomeWeakRef(s_obj.classdef) + +BUILTIN_ANALYZERS[weakref.ref] = weakref_ref + +#________________________________ # non-gc objects def robjmodel_free_non_gc_object(obj): Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Mon Sep 10 19:39:28 2007 @@ -492,6 +492,17 @@ s_ImpossibleValue = SomeImpossibleValue() # ____________________________________________________________ +# weakrefs + +class SomeWeakRef(SomeObject): + immutable = True + def __init__(self, classdef): + self.classdef = classdef + + def can_be_none(self): + return False + +# ____________________________________________________________ # memory addresses from pypy.rpython.lltypesystem import llmemory Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Mon Sep 10 19:39:28 2007 @@ -2804,6 +2804,31 @@ s = a.build_types(f, [int]) assert isinstance(s, annmodel.SomeInteger) + def test_weakref(self): + import weakref + + class A: + pass + class B(A): + pass + class C(A): + pass + + def f(n): + if n: + b = B() + b.hello = 42 + r = weakref.ref(b) + else: + c = C() + c.hello = 64 + r = weakref.ref(c) + return r().hello + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeInteger) + assert not s.is_constant() + def g(n): return [0,1,2,n] Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Mon Sep 10 19:39:28 2007 @@ -7,7 +7,7 @@ SomeDict, SomeUnicodeCodePoint, SomeTuple, SomeImpossibleValue, \ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ - SomeCTypesObject, s_ImpossibleValue, s_Bool, \ + SomeCTypesObject, SomeWeakRef, s_ImpossibleValue, s_Bool, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ SomeGenericCallable from pypy.annotation.bookkeeper import getbookkeeper @@ -751,6 +751,13 @@ return entry.compute_result_annotation(*args_s) #_________________________________________ +# weakrefs + +class __extend__(SomeWeakRef): + def simple_call(s_wrf): + return SomeInstance(s_wrf.classdef, can_be_None=True) + +#_________________________________________ # memory addresses from pypy.rpython.memory import lladdress From simonb at codespeak.net Mon Sep 10 19:56:33 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Mon, 10 Sep 2007 19:56:33 +0200 (CEST) Subject: [pypy-svn] r46455 - pypy/dist/pypy/rpython/numpy Message-ID: <20070910175633.0D7798100@code0.codespeak.net> Author: simonb Date: Mon Sep 10 19:56:33 2007 New Revision: 46455 Modified: pypy/dist/pypy/rpython/numpy/rarray.py Log: start to simplify code by using adtmeths Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Mon Sep 10 19:56:33 2007 @@ -1,24 +1,25 @@ +from pypy.rpython.error import TyperError +from pypy.annotation.model import SomeObject, SomeInteger +from pypy.rpython.numpy.aarray import SomeArray +from pypy.annotation.pairtype import pairtype, pair +from pypy.rlib.unroll import unrolling_iterable +from pypy.annotation import listdef +from pypy.rlib.objectmodel import debug_assert +from pypy.rpython.annlowlevel import ADTInterface +from pypy.rpython.memory.lltypelayout import sizeof from pypy.rpython.rmodel import Repr, FloatRepr, inputconst from pypy.rpython.rrange import AbstractRangeRepr from pypy.rpython.rint import IntegerRepr from pypy.rpython.rlist import AbstractBaseListRepr +from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython.rtuple import AbstractTupleRepr -from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem import lltype, llmemory, rtuple from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE -from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython.lltypesystem.lltype import \ GcArray, GcStruct, Number, Primitive, Signed, Ptr, Unsigned, Void, FixedSizeArray, Bool,\ - GcForwardReference, malloc, direct_arrayitems, direct_ptradd, nullptr + GcForwardReference, malloc, direct_arrayitems, direct_ptradd, nullptr, typeMethod from pypy.rpython.lltypesystem.rffi import cast from pypy.rpython.lltypesystem.rtuple import TupleRepr -from pypy.annotation.model import SomeObject, SomeInteger -from pypy.rpython.numpy.aarray import SomeArray -from pypy.annotation.pairtype import pairtype, pair -from pypy.rlib.unroll import unrolling_iterable -from pypy.annotation import listdef -from pypy.rpython.memory.lltypelayout import sizeof -from pypy.rlib.objectmodel import debug_assert def gen_build_from_shape(ndim, zero=False): unrolling_dims = unrolling_iterable(reversed(range(ndim))) @@ -66,6 +67,33 @@ NPY_INTP = Signed # XX index type (see Py_intptr_t) def ARRAY_ITER(ARRAY, INDEXARRAY): + ndim = dim_of_ARRAY(ARRAY) + + unroll_ndim = unrolling_iterable(range(ndim)) + def ll_iter_reset(it, dataptr): + it.index = 0 + it.dataptr = dataptr + for i in unroll_ndim: + it.coordinates[i] = 0 + ll_iter_reset._always_inline_ = True + + unroll_ndim_rev = unrolling_iterable(reversed(range(ndim))) + def ll_iter_next(it): + it.index += 1 + for i in unroll_ndim_rev: + if it.coordinates[i] < it.dims_m1[i]: + it.coordinates[i] += 1 + it.dataptr = direct_ptradd(it.dataptr, it.strides[i]) + break + it.coordinates[i] = 0 + it.dataptr = direct_ptradd(it.dataptr, -it.backstrides[i]) + ll_iter_next._always_inline_ = True + + ADTIIter = ADTInterface(None, { +# 'll_new':['SELF', ARRAY, ARRAY], + 'll_reset': (['self', ARRAY.TO.dataptr], Void), + 'll_next': (['self'], Void), + }) ITER = Ptr( GcStruct("array_iter", ("nd_m1", Signed), # number of dimensions - 1 @@ -79,6 +107,10 @@ #("ao", ARRAY), # not needed.. ("dataptr", ARRAY.TO.dataptr), # pointer to current item #("contiguous", Bool), + adtmeths = ADTIIter({ + 'll_next': ll_iter_next, + 'll_reset': ll_iter_reset, + }), )) return ITER @@ -91,16 +123,8 @@ def gen_iter_funcs(ndim): unroll_ndim = unrolling_iterable(range(ndim)) - unroll_ndim_rev = unrolling_iterable(reversed(range(ndim))) - def ll_iter_reset(it, dataptr): - it.index = 0 - it.dataptr = dataptr - for i in unroll_ndim: - it.coordinates[i] = 0 - ll_iter_reset._always_inline_ = True - - def ll_iter_new(ITER, ao, target_ao, iter_reset, iter_broadcast_to_shape): + def ll_iter_new(ITER, ao, target_ao, iter_broadcast_to_shape): assert ao.dataptr # Suffix of ao.shape must match target_ao.shape # (suffix starts at the first non-1 entry in ao.shape.) @@ -115,7 +139,7 @@ broadcast = True i += 1 if broadcast: - return iter_broadcast_to_shape(ITER, ao, target_ao, iter_reset) + return iter_broadcast_to_shape(ITER, ao, target_ao) it = malloc(ITER) it.nd_m1 = ndim - 1 it.size = ll_mul_list(ao.shape, ndim) @@ -126,11 +150,11 @@ it.backstrides[i] = it.strides[i] * it.dims_m1[i] #if i > 0: #it.factors[nd-i-1] = it.factors[nd]*ao.shape[nd-i] - iter_reset(it, ao.dataptr) + it.ll_reset(ao.dataptr) return it ll_iter_new._always_inline_ = True - def ll_iter_broadcast_to_shape(ITER, ao, target_ao, iter_reset): + def ll_iter_broadcast_to_shape(ITER, ao, target_ao): "iterate over but broadcast to the shape of " debug_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim") delta = j = ndim - ao.ndim @@ -154,25 +178,11 @@ it.backstrides[i] = it.strides[i] * it.dims_m1[i] #if i > 0: #it.factors[ndim-i-1] = it.factors[nd-i]*shape[ndim-i] - iter_reset(it, ao.dataptr) + it.ll_reset(ao.dataptr) return it ll_iter_broadcast_to_shape._always_inline_ = True - # XXX - # I think this is the only function that needs to be - # generated-per-ndim: - def ll_iter_next(it): - it.index += 1 - for i in unroll_ndim_rev: - if it.coordinates[i] < it.dims_m1[i]: - it.coordinates[i] += 1 - it.dataptr = direct_ptradd(it.dataptr, it.strides[i]) - break - it.coordinates[i] = 0 - it.dataptr = direct_ptradd(it.dataptr, -it.backstrides[i]) - ll_iter_next._always_inline_ = True - - return ll_iter_new, ll_iter_reset, ll_iter_broadcast_to_shape, ll_iter_next + return ll_iter_new, ll_iter_broadcast_to_shape def ll_unary_op(p0, p1, op=lambda x:x): p0[0] = op(p1[0]) @@ -181,19 +191,14 @@ p0[0] = op(p1[0], p2[0]) -def ll_array_set(it0, it1, iter_next): +def ll_array_set(it0, it1): if it0.size == 0: return # empty LHS.. debug_assert(it0.size == it1.size, "it0.size == it1.size") while it0.index < it0.size: it0.dataptr[0] = it1.dataptr[0] - iter_next(it0) - iter_next(it1) - -def ll_array_set1(value, it, iter_next): - while it.index < it.size: - it.dataptr[0] = value - iter_next(it) + it0.ll_next() + it1.ll_next() def dim_of_ITER(ITER): return ITER.TO.coordinates.length @@ -201,11 +206,15 @@ def dim_of_ARRAY(ARRAY): return ARRAY.TO.shape.length -class ArrayIterRepr(Repr): +class ArrayIterRepr(Repr): # XX ? def __init__(self, rtyper, s_iter): self.s_iter = s_iter self.lowleveltype = self.ITER +ADTIArray = ADTInterface(None, { + 'll_allocate': (['SELF', Signed], 'self'), +}) + class ArrayRepr(Repr): def make_types(cls, ndim, ITEM): DATA_PTR = Ptr(FixedSizeArray(ITEM, 1)) @@ -218,6 +227,9 @@ ("shape", INDEXARRAY), # size in each dimension ("strides", INDEXARRAY), # elements to jump to get to the # next element in each dimension + adtmeths = ADTIArray({ + "ll_allocate" : ll_allocate, + }), ) ARRAY = Ptr(STRUCT) return ARRAY, INDEXARRAY @@ -301,30 +313,28 @@ key = self.__class__, self.typecode, self.ndim return key -def ll_array_binop(it0, it1, it2, iter_next, binop): +def ll_array_binop(it0, it1, it2, binop): debug_assert(it0.size == it1.size, "it0.size == it1.size") debug_assert(it1.size == it2.size, "it0.size == it1.size") while it0.index < it0.size: it0.dataptr[0] = binop(it1.dataptr[0], it2.dataptr[0]) - iter_next(it0) - iter_next(it1) - iter_next(it2) + it0.ll_next() + it1.ll_next() + it2.ll_next() def _rtype_binop(r_array0, r_array1, r_array2, v_array1, v_array2, hop, binop): cARRAY = hop.inputconst(Void, r_array0.ARRAY.TO) # We build a contiguous "result" array # from the largest of the two args: v_array0 = hop.gendirectcall(ll_build_like, cARRAY, v_array1, v_array2) - iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(r_array0.ndim) + iter_new, iter_broadcast = gen_iter_funcs(r_array0.ndim) cITER = hop.inputconst(Void, r_array0.ITER.TO) - creset = hop.inputconst(Void, iter_reset) cbroadcast = hop.inputconst(Void, iter_broadcast) - cnext = hop.inputconst(Void, iter_next) - v_it0 = hop.gendirectcall(iter_new, cITER, v_array0, v_array0, creset, cbroadcast) - v_it1 = hop.gendirectcall(iter_new, cITER, v_array1, v_array0, creset, cbroadcast) - v_it2 = hop.gendirectcall(iter_new, cITER, v_array2, v_array0, creset, cbroadcast) + v_it0 = hop.gendirectcall(iter_new, cITER, v_array0, v_array0, cbroadcast) + v_it1 = hop.gendirectcall(iter_new, cITER, v_array1, v_array0, cbroadcast) + v_it2 = hop.gendirectcall(iter_new, cITER, v_array2, v_array0, cbroadcast) cbinop = hop.inputconst(Void, binop) - hop.gendirectcall(ll_array_binop, v_it0, v_it1, v_it2, cnext, cbinop) + hop.gendirectcall(ll_array_binop, v_it0, v_it1, v_it2, cbinop) return v_array0 def rtype_binop((r_array1, r_array2), hop, binop): @@ -355,14 +365,14 @@ def gen_getset_item(ndim): unrolling_dims = unrolling_iterable(range(ndim)) def ll_get_item(ARRAY, ao, tpl): - array = ll_allocate(ARRAY, ndim) + array = ARRAY.ll_allocate(ndim) idx = 0 for i in unrolling_dims: idx += ao.strides[i] * getattr(tpl, 'item%d'%i) return ao.data[idx] def ll_set_item(ARRAY, ao, tpl, value): - array = ll_allocate(ARRAY, ndim) + array = ARRAY.ll_allocate(ndim) idx = 0 for i in unrolling_dims: idx += ao.strides[i] * getattr(tpl, 'item%d'%i) @@ -380,7 +390,7 @@ unroll_r_tuple = unrolling_iterable(enumerate(r_tuple.items_r)) rslice = hop.rtyper.type_system.rslice def ll_get_view(ARRAY, ao, tpl): - array = ll_allocate(ARRAY, ndim) + array = ARRAY.ll_allocate(ndim) dataptr = direct_arrayitems(ao.data) src_i = 0 tgt_i = 0 @@ -511,12 +521,10 @@ cARRAY = hop.inputconst(Void, r_view.ARRAY.TO) get_view = gen_get_view(r_array, r_tuple, hop) v_view = hop.gendirectcall(get_view, cARRAY, v_array, v_tuple) - iter_new, iter_reset, iter_broadcast, iter_next = gen_iter_funcs(ndim) - creset = hop.inputconst(Void, iter_reset) + iter_new, iter_broadcast = gen_iter_funcs(ndim) cbroadcast = hop.inputconst(Void, iter_broadcast) - cnext = hop.inputconst(Void, iter_next) cITER = hop.inputconst(Void, r_view.ITER.TO) - v_it0 = hop.gendirectcall(iter_new, cITER, v_view, v_view, creset, cbroadcast) + v_it0 = hop.gendirectcall(iter_new, cITER, v_view, v_view, cbroadcast) if isinstance(r_item, ArrayRepr): source_ndim = r_item.ndim elif isinstance(r_item.lowleveltype, Primitive): @@ -529,8 +537,8 @@ else: raise TypeError("bad item: %s"%r_item) assert source_ndim <= ndim - v_it1 = hop.gendirectcall(iter_new, cITER, v_item, v_view, creset, cbroadcast) - return hop.gendirectcall(ll_array_set, v_it0, v_it1, cnext) + v_it1 = hop.gendirectcall(iter_new, cITER, v_item, v_view, cbroadcast) + return hop.gendirectcall(ll_array_set, v_it0, v_it1) class __extend__(pairtype(ArrayRepr, ArrayRepr)): def convert_from_to((r_array0, r_array1), v, llops): @@ -557,9 +565,10 @@ array.data = nullptr(ARRAY.data.TO) array.dataptr = nullptr(ARRAY.dataptr.TO) return array +ll_allocate = typeMethod(ll_allocate) def ll_build_from_size(ARRAY, size, _malloc): - array = ll_allocate(ARRAY, 1) + array = ARRAY.ll_allocate(1) array.shape[0] = size array.strides[0] = 1 array.data = _malloc(ARRAY.data.TO, size) @@ -568,7 +577,7 @@ def ll_build_from_list(ARRAY, lst): size = lst.ll_length() - array = ll_allocate(ARRAY, 1) + array = ARRAY.ll_allocate(1) array.shape[0] = size array.strides[0] = 1 array.data = malloc(ARRAY.data.TO, size) @@ -582,7 +591,7 @@ def ll_build_alias_to_list(ARRAY, lst): # This should only be used for temporary calculations size = lst.ll_length() - array = ll_allocate(ARRAY, 1) + array = ARRAY.ll_allocate(1) array.shape[0] = size array.strides[0] = 1 # Well.. this doesn't work (because array.data has nolength ?) @@ -591,7 +600,7 @@ return array def ll_build_from_scalar(ARRAY, value): - array = ll_allocate(ARRAY, 1) + array = ARRAY.ll_allocate(1) array.shape[0] = 1 array.strides[0] = 1 array.data = malloc(ARRAY.data.TO, 1) @@ -600,7 +609,7 @@ return array def ll_build_alias(ARRAY, ao): - array = ll_allocate(ARRAY, ao.ndim) + array = ARRAY.ll_allocate(ao.ndim) array.data = ao.data # alias data for i in range(ao.ndim): array.shape[i] = ao.shape[i] @@ -612,7 +621,7 @@ # Build with shape from the largest of array0 or array1. # Note we cannot take the union of array0 and array1. ndim = max(array0.ndim, array1.ndim) - array = ll_allocate(ARRAY, ndim) + array = ARRAY.ll_allocate(ndim) sz0 = ll_mul_list(array0.shape, array0.ndim) sz1 = ll_mul_list(array1.shape, array1.ndim) sz = max(sz0, sz1) @@ -652,7 +661,7 @@ def ll_transpose(ARRAY, ao): ndim = ao.ndim - array = ll_allocate(ARRAY, ndim) + array = ARRAY.ll_allocate(ndim) array.data = ao.data # alias data for i in range(ndim): array.shape[i] = ao.shape[ndim-i-1] From arigo at codespeak.net Mon Sep 10 20:08:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Sep 2007 20:08:06 +0200 (CEST) Subject: [pypy-svn] r46458 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070910180806.E4E968105@code0.codespeak.net> Author: arigo Date: Mon Sep 10 20:08:06 2007 New Revision: 46458 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Log: Basic operations in llmemory. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Mon Sep 10 20:08:06 2007 @@ -515,7 +515,7 @@ s = 'NULL' else: s = str(self.ref) - return '' % (s,) + return '<%s %s>' % (self.__class__.__name__, s) def cast_to_int(self): # this is not always the behaviour that is really happening # but make sure that nobody depends on it @@ -540,6 +540,26 @@ # ____________________________________________________________ +WeakGcRefOpaque = lltype.OpaqueType('WeakGcRef') + +def weakgcref_init(wropaque, obj): + PTRTYPE = lltype.typeOf(obj) + assert isinstance(PTRTYPE, lltype.Ptr) + assert PTRTYPE.TO._gckind == 'gc' + wropaque._obj.ref = weakref.ref(lltype.normalizeptr(obj)) + +def weakgcref_get(PTRTYPE, wropaque): + assert isinstance(PTRTYPE, lltype.Ptr) + assert PTRTYPE.TO._gckind == 'gc' + assert lltype.typeOf(wropaque) == lltype.Ptr(WeakGcRefOpaque) + p = wropaque._obj.ref() + if p is None: + return lltype.nullptr(PTRTYPE.TO) + else: + return lltype.cast_pointer(PTRTYPE, p) + +# ____________________________________________________________ + def raw_malloc(size): if not isinstance(size, AddressOffset): raise NotImplementedError(size) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Mon Sep 10 20:08:06 2007 @@ -573,6 +573,19 @@ assert s2_ptr.x == 2 # release(start) +def test_weakref(): + S = lltype.GcStruct('S', ('x',lltype.Signed)) + s = lltype.malloc(S) + o = lltype.malloc(WeakGcRefOpaque, flavor='raw') + weakgcref_init(o, s) + assert weakgcref_get(lltype.Ptr(S), o) == s + assert weakgcref_get(lltype.Ptr(S), o) == s + del s + import gc; gc.collect() + assert weakgcref_get(lltype.Ptr(S), o) == lltype.nullptr(S) + lltype.free(o, flavor='raw') + py.test.raises(RuntimeError, weakgcref_get, lltype.Ptr(S), o) + class TestWeakAddressLLinterp(object): def test_null(self): from pypy.rlib.objectmodel import cast_weakgcaddress_to_object From arigo at codespeak.net Tue Sep 11 10:44:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 10:44:22 +0200 (CEST) Subject: [pypy-svn] r46459 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070911084422.A76B980D2@code0.codespeak.net> Author: arigo Date: Tue Sep 11 10:44:21 2007 New Revision: 46459 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Log: A more direct approach to weakrefs, without using opaques. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Tue Sep 11 10:44:21 2007 @@ -540,24 +540,30 @@ # ____________________________________________________________ -WeakGcRefOpaque = lltype.OpaqueType('WeakGcRef') +class fakeweakref(fakeweakaddress): + pass # inheriting only to copy all methods -def weakgcref_init(wropaque, obj): +WeakRef = lltype.Primitive("WeakRef", fakeweakref(None)) + +def weakref_create(obj): PTRTYPE = lltype.typeOf(obj) assert isinstance(PTRTYPE, lltype.Ptr) assert PTRTYPE.TO._gckind == 'gc' - wropaque._obj.ref = weakref.ref(lltype.normalizeptr(obj)) + return fakeweakref(lltype.normalizeptr(obj)) -def weakgcref_get(PTRTYPE, wropaque): +def weakref_deref(PTRTYPE, wref): assert isinstance(PTRTYPE, lltype.Ptr) assert PTRTYPE.TO._gckind == 'gc' - assert lltype.typeOf(wropaque) == lltype.Ptr(WeakGcRefOpaque) - p = wropaque._obj.ref() + assert lltype.typeOf(wref) == WeakRef + p = wref.get() if p is None: return lltype.nullptr(PTRTYPE.TO) else: return lltype.cast_pointer(PTRTYPE, p) +fakeweakref._TYPE = WeakRef +WEAKREFNULL = fakeweakref(None) + # ____________________________________________________________ def raw_malloc(size): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Tue Sep 11 10:44:21 2007 @@ -576,15 +576,12 @@ def test_weakref(): S = lltype.GcStruct('S', ('x',lltype.Signed)) s = lltype.malloc(S) - o = lltype.malloc(WeakGcRefOpaque, flavor='raw') - weakgcref_init(o, s) - assert weakgcref_get(lltype.Ptr(S), o) == s - assert weakgcref_get(lltype.Ptr(S), o) == s + w = weakref_create(s) + assert weakref_deref(lltype.Ptr(S), w) == s + assert weakref_deref(lltype.Ptr(S), w) == s del s import gc; gc.collect() - assert weakgcref_get(lltype.Ptr(S), o) == lltype.nullptr(S) - lltype.free(o, flavor='raw') - py.test.raises(RuntimeError, weakgcref_get, lltype.Ptr(S), o) + assert weakref_deref(lltype.Ptr(S), w) == lltype.nullptr(S) class TestWeakAddressLLinterp(object): def test_null(self): From arigo at codespeak.net Tue Sep 11 11:11:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 11:11:56 +0200 (CEST) Subject: [pypy-svn] r46460 - in pypy/dist/pypy: annotation rpython rpython/lltypesystem rpython/test Message-ID: <20070911091156.7D6E38104@code0.codespeak.net> Author: arigo Date: Tue Sep 11 11:11:55 2007 New Revision: 46460 Added: pypy/dist/pypy/rpython/rweakref.py (contents, props changed) pypy/dist/pypy/rpython/test/test_rweakref.py (contents, props changed) Modified: pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rtyper.py Log: Annotation and rtyping for low-level WeakRefs. Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Tue Sep 11 11:11:55 2007 @@ -8,7 +8,8 @@ from pypy.annotation.model import SomeUnicodeCodePoint, SomeAddress from pypy.annotation.model import SomeFloat, SomeWeakGcAddress, unionof from pypy.annotation.model import SomePBC, SomeInstance, SomeDict -from pypy.annotation.model import SomeExternalObject, SomeWeakRef +from pypy.annotation.model import SomeExternalObject +from pypy.annotation.model import SomeWeakRef, SomeLLWeakRef from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata from pypy.annotation.model import s_ImpossibleValue @@ -580,6 +581,28 @@ BUILTIN_ANALYZERS[weakref.ref] = weakref_ref +def llweakref_create(s_obj): + if (not isinstance(s_obj, SomePtr) or + s_obj.ll_ptrtype.TO._gckind != 'gc'): + raise Exception("bad type for argument to weakref_create(): %r" % ( + s_obj,)) + return SomeLLWeakRef() + +def llweakref_deref(s_ptrtype, s_wref): + if not (s_ptrtype.is_constant() and + isinstance(s_ptrtype.const, lltype.Ptr) and + s_ptrtype.const.TO._gckind == 'gc'): + raise Exception("weakref_deref() arg 1 must be a constant " + "ptr type, got %s" % (s_ptrtype,)) + if not isinstance(s_wref, SomeLLWeakRef): + raise Exception("weakref_deref() arg 2 must be a llweakref, " + "got %s" % (s_wref,)) + return SomePtr(s_ptrtype.const) + +from pypy.rpython.lltypesystem import llmemory +BUILTIN_ANALYZERS[llmemory.weakref_create] = llweakref_create +BUILTIN_ANALYZERS[llmemory.weakref_deref ] = llweakref_deref + #________________________________ # non-gc objects Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Tue Sep 11 11:11:55 2007 @@ -33,7 +33,7 @@ from pypy.annotation.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, base_int -import inspect +import inspect, weakref from sys import maxint from pypy.annotation.description import FunctionDesc @@ -495,6 +495,7 @@ # weakrefs class SomeWeakRef(SomeObject): + knowntype = weakref.ref immutable = True def __init__(self, classdef): self.classdef = classdef @@ -502,6 +503,12 @@ def can_be_none(self): return False +class SomeLLWeakRef(SomeObject): + immutable = True + + def can_be_none(self): + return False + # ____________________________________________________________ # memory addresses @@ -583,6 +590,7 @@ (SomeUnicodeCodePoint(), lltype.UniChar), (SomeAddress(), llmemory.Address), (SomeWeakGcAddress(), llmemory.WeakGcAddress), + (SomeLLWeakRef(), llmemory.WeakRef), ] def annotation_to_lltype(s_val, info=None): Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Sep 11 11:11:55 2007 @@ -702,6 +702,13 @@ return lltype.cast_opaque_ptr(RESTYPE, obj) op_cast_opaque_ptr.need_result_type = True + def op_weakref_create(self, obj): + return llmemory.weakref_create(obj) + + def op_weakref_deref(self, PTRTYPE, obj): + return llmemory.weakref_deref(PTRTYPE, obj) + op_weakref_deref.need_result_type = True + def op_gc__collect(self): import gc gc.collect() Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Tue Sep 11 11:11:55 2007 @@ -366,6 +366,8 @@ 'cast_weakadr_to_int': LLOp(canfold=True), 'cast_adr_to_int': LLOp(canfold=True), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp + 'weakref_create': LLOp(sideeffects=False), + 'weakref_deref': LLOp(sideeffects=False), # __________ used by the JIT ________ Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Tue Sep 11 11:11:55 2007 @@ -594,6 +594,22 @@ BUILTIN_TYPER[llmemory.offsetof] = rtype_offsetof # _________________________________________________________________ +# weakrefs + +def rtype_weakref_create(hop): + vlist = hop.inputargs(hop.args_r[0]) + hop.exception_cannot_occur() + return hop.genop('weakref_create', vlist, resulttype=llmemory.WeakRef) + +def rtype_weakref_deref(hop): + c_ptrtype, v_wref = hop.inputargs(lltype.Void, llmemory.WeakRef) + hop.exception_cannot_occur() + return hop.genop('weakref_deref', [v_wref], resulttype=c_ptrtype.value) + +BUILTIN_TYPER[llmemory.weakref_create] = rtype_weakref_create +BUILTIN_TYPER[llmemory.weakref_deref ] = rtype_weakref_deref + +# _________________________________________________________________ # non-gc objects def rtype_free_non_gc_object(hop): Modified: pypy/dist/pypy/rpython/rtyper.py ============================================================================== --- pypy/dist/pypy/rpython/rtyper.py (original) +++ pypy/dist/pypy/rpython/rtyper.py Tue Sep 11 11:11:55 2007 @@ -947,5 +947,6 @@ from pypy.rpython import rexternalobj from pypy.rpython import rptr from pypy.rpython import rgeneric +from pypy.rpython import rweakref from pypy.rpython import raddress # memory addresses from pypy.rpython.ootypesystem import rootype Added: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/rweakref.py Tue Sep 11 11:11:55 2007 @@ -0,0 +1,14 @@ +from pypy.annotation import model as annmodel +from pypy.rpython.rmodel import Repr +from pypy.rpython.lltypesystem import lltype, llmemory + + +class __extend__(annmodel.SomeLLWeakRef): + def rtyper_makerepr(self, rtyper): + return LLWeakRefRepr() + def rtyper_makekey(self): + return self.__class__, + + +class LLWeakRefRepr(Repr): + lowleveltype = llmemory.WeakRef Added: pypy/dist/pypy/rpython/test/test_rweakref.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/test/test_rweakref.py Tue Sep 11 11:11:55 2007 @@ -0,0 +1,20 @@ +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.test.test_llinterp import interpret + + +def test_ll_weakref(): + S = lltype.GcStruct('S', ('x',lltype.Signed)) + def g(): + s = lltype.malloc(S) + w = llmemory.weakref_create(s) + assert llmemory.weakref_deref(lltype.Ptr(S), w) == s + assert llmemory.weakref_deref(lltype.Ptr(S), w) == s + return w # 's' is forgotten here + def f(): + w = g() + rgc.collect() + return llmemory.weakref_deref(lltype.Ptr(S), w) + + res = interpret(f, []) + assert res == lltype.nullptr(S) From arigo at codespeak.net Tue Sep 11 11:41:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 11:41:59 +0200 (CEST) Subject: [pypy-svn] r46461 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070911094159.2B26F8107@code0.codespeak.net> Author: arigo Date: Tue Sep 11 11:41:58 2007 New Revision: 46461 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Log: Fix fakeweakaddress to resist to pointer casts. This fixes fakeweakref too. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Tue Sep 11 11:41:58 2007 @@ -491,8 +491,11 @@ # ____________________________________________________________ class fakeweakaddress(object): + # XXX convoluted code to support both lltype._ptr and simulatorptr def __init__(self, ob): if ob is not None: + if isinstance(ob, lltype._ptr): + ob = lltype.normalizeptr(ob)._obj self.ref = weakref.ref(ob) # umpf from pypy.rpython.memory import lltypesimulation @@ -509,6 +512,8 @@ # xxx stop-gap #if ob is None: # raise DanglingPointerError + if isinstance(ob, lltype._container): + ob = ob._as_ptr() return ob def __repr__(self): if self.ref is None: @@ -525,7 +530,6 @@ fakeweakaddress(None)) def cast_ptr_to_weakadr(obj): - # XXX this is missing the normalizations done by _ptr._cast_to_adr() assert isinstance(lltype.typeOf(obj), lltype.Ptr) return fakeweakaddress(obj) @@ -533,7 +537,8 @@ result = adr.get() if result is None: return lltype.nullptr(EXPECTED_TYPE.TO) - return result + else: + return lltype.cast_pointer(EXPECTED_TYPE, result) fakeweakaddress._TYPE = WeakGcAddress WEAKNULL = fakeweakaddress(None) @@ -549,7 +554,7 @@ PTRTYPE = lltype.typeOf(obj) assert isinstance(PTRTYPE, lltype.Ptr) assert PTRTYPE.TO._gckind == 'gc' - return fakeweakref(lltype.normalizeptr(obj)) + return fakeweakref(obj) def weakref_deref(PTRTYPE, wref): assert isinstance(PTRTYPE, lltype.Ptr) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Tue Sep 11 11:41:58 2007 @@ -169,6 +169,23 @@ res = interpret(f, []) assert res +def test_weak_casts_lifetime(): + T = lltype.GcStruct("T", ("x", lltype.Signed)) + S = lltype.GcStruct("S", ("t", T)) + Sptr = lltype.Ptr(S) + def g(): + s1 = lltype.malloc(S) + adr = cast_ptr_to_weakadr(s1) + s2 = cast_weakadr_to_ptr(adr, Sptr) + assert s1 == s2 + return adr, lltype.cast_pointer(lltype.Ptr(T), s2) # s1, s2 go away + def f(): + adr, t = g() + t2 = cast_weakadr_to_ptr(adr, lltype.Ptr(T)) + return t2 == t + res = interpret(f, []) + assert res + def test_fakeaccessor(): S = lltype.GcStruct("S", ("x", lltype.Signed), ("y", lltype.Signed)) s = lltype.malloc(S) @@ -574,14 +591,23 @@ # release(start) def test_weakref(): - S = lltype.GcStruct('S', ('x',lltype.Signed)) + S1 = lltype.GcStruct('S1', ('x',lltype.Signed)) + S = lltype.GcStruct('S', ('s1', S1)) s = lltype.malloc(S) + s1 = lltype.cast_pointer(lltype.Ptr(S1), s) w = weakref_create(s) assert weakref_deref(lltype.Ptr(S), w) == s - assert weakref_deref(lltype.Ptr(S), w) == s + assert weakref_deref(lltype.Ptr(S1), w) == s1 + # check that the weakref stays alive even if there are only + # cast_pointer'ed references around del s import gc; gc.collect() + assert weakref_deref(lltype.Ptr(S1), w) == s1 + # now really kill the structure + del s1 + import gc; gc.collect() assert weakref_deref(lltype.Ptr(S), w) == lltype.nullptr(S) + assert weakref_deref(lltype.Ptr(S1), w) == lltype.nullptr(S1) class TestWeakAddressLLinterp(object): def test_null(self): From arigo at codespeak.net Tue Sep 11 11:43:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 11:43:05 +0200 (CEST) Subject: [pypy-svn] r46462 - in pypy/dist/pypy/rpython: . test Message-ID: <20070911094305.76E4A8107@code0.codespeak.net> Author: arigo Date: Tue Sep 11 11:43:05 2007 New Revision: 46462 Modified: pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rweakref.py pypy/dist/pypy/rpython/test/test_rweakref.py Log: RTyping of RPython-level weakrefs (lltypesystem only in this check-in). Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Tue Sep 11 11:43:05 2007 @@ -596,7 +596,10 @@ # _________________________________________________________________ # weakrefs +import weakref + def rtype_weakref_create(hop): + # Note: this code also works for the RPython-level calls 'weakref.ref(x)'. vlist = hop.inputargs(hop.args_r[0]) hop.exception_cannot_occur() return hop.genop('weakref_create', vlist, resulttype=llmemory.WeakRef) @@ -606,6 +609,7 @@ hop.exception_cannot_occur() return hop.genop('weakref_deref', [v_wref], resulttype=c_ptrtype.value) +BUILTIN_TYPER[weakref.ref] = rtype_weakref_create BUILTIN_TYPER[llmemory.weakref_create] = rtype_weakref_create BUILTIN_TYPER[llmemory.weakref_deref ] = rtype_weakref_deref Modified: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/rweakref.py (original) +++ pypy/dist/pypy/rpython/rweakref.py Tue Sep 11 11:43:05 2007 @@ -9,6 +9,24 @@ def rtyper_makekey(self): return self.__class__, - class LLWeakRefRepr(Repr): lowleveltype = llmemory.WeakRef + +# ____________________________________________________________ +# +# RPython-level weakrefs + +class __extend__(annmodel.SomeWeakRef): + def rtyper_makerepr(self, rtyper): + return WeakRefRepr() + def rtyper_makekey(self): + return self.__class__, + + +class WeakRefRepr(Repr): + lowleveltype = llmemory.WeakRef + + def rtype_simple_call(self, hop): + v_wref, = hop.inputargs(self) + hop.exception_cannot_occur() + return hop.genop('weakref_deref', [v_wref], resulttype=hop.r_result) Modified: pypy/dist/pypy/rpython/test/test_rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rweakref.py (original) +++ pypy/dist/pypy/rpython/test/test_rweakref.py Tue Sep 11 11:43:05 2007 @@ -1,3 +1,4 @@ +import weakref from pypy.rlib import rgc from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.test.test_llinterp import interpret @@ -18,3 +19,29 @@ res = interpret(f, []) assert res == lltype.nullptr(S) + + +def test_weakref_simple(): + class A: + pass + class B(A): + pass + class C(A): + pass + + def f(n): + if n: + x = B() + x.hello = 42 + r = weakref.ref(x) + else: + x = C() + x.hello = 64 + r = weakref.ref(x) + return r().hello, x # returns 'x' too, to keep it alive + + res = interpret(f, [1]) + assert res.item0 == 42 + + res = interpret(f, [0]) + assert res.item0 == 64 From arigo at codespeak.net Tue Sep 11 12:14:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 12:14:49 +0200 (CEST) Subject: [pypy-svn] r46464 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20070911101449.259A18100@code0.codespeak.net> Author: arigo Date: Tue Sep 11 12:14:48 2007 New Revision: 46464 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py Log: For compatibility with existing code and tests all around the place, re-emulate the previous interface about 'ref'. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Tue Sep 11 12:14:48 2007 @@ -496,7 +496,8 @@ if ob is not None: if isinstance(ob, lltype._ptr): ob = lltype.normalizeptr(ob)._obj - self.ref = weakref.ref(ob) + self.obref = weakref.ref(ob) + self.ref = self.get # backward compatibility # umpf from pypy.rpython.memory import lltypesimulation if isinstance(ob, (lltype._ptr,lltypesimulation.simulatorptr)): @@ -504,11 +505,12 @@ else: self.id = id(ob) else: + self.obref = None self.ref = None def get(self): - if self.ref is None: + if self.obref is None: return None - ob = self.ref() + ob = self.obref() # xxx stop-gap #if ob is None: # raise DanglingPointerError @@ -516,10 +518,10 @@ ob = ob._as_ptr() return ob def __repr__(self): - if self.ref is None: + if self.obref is None: s = 'NULL' else: - s = str(self.ref) + s = str(self.obref) return '<%s %s>' % (self.__class__.__name__, s) def cast_to_int(self): # this is not always the behaviour that is really happening From arigo at codespeak.net Tue Sep 11 12:17:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 12:17:50 +0200 (CEST) Subject: [pypy-svn] r46466 - in pypy/dist/pypy/rpython: . test Message-ID: <20070911101750.515E38101@code0.codespeak.net> Author: arigo Date: Tue Sep 11 12:17:49 2007 New Revision: 46466 Modified: pypy/dist/pypy/rpython/rweakref.py pypy/dist/pypy/rpython/test/test_rweakref.py Log: Prebuilt non-dead RPython-level weakrefs... Modified: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/rweakref.py (original) +++ pypy/dist/pypy/rpython/rweakref.py Tue Sep 11 12:17:49 2007 @@ -1,5 +1,8 @@ +import weakref from pypy.annotation import model as annmodel +from pypy.objspace.flow.model import Constant from pypy.rpython.rmodel import Repr +from pypy.rpython.rclass import getinstancerepr from pypy.rpython.lltypesystem import lltype, llmemory @@ -18,7 +21,7 @@ class __extend__(annmodel.SomeWeakRef): def rtyper_makerepr(self, rtyper): - return WeakRefRepr() + return WeakRefRepr(rtyper) def rtyper_makekey(self): return self.__class__, @@ -26,7 +29,23 @@ class WeakRefRepr(Repr): lowleveltype = llmemory.WeakRef + def __init__(self, rtyper): + self.rtyper = rtyper + def rtype_simple_call(self, hop): v_wref, = hop.inputargs(self) hop.exception_cannot_occur() return hop.genop('weakref_deref', [v_wref], resulttype=hop.r_result) + + def convert_const(self, value): + assert isinstance(value, weakref.ref) + instance = value() + bk = self.rtyper.annotator.bookkeeper + # obscure! if the annotator hasn't seen this object before, + # we don't want to look at it now (confusion tends to result). + if instance is None or not bk.have_seen(instance): + return llmemory.WeakRef._defl() + else: + repr = self.rtyper.bindingrepr(Constant(instance)) + llinstance = repr.convert_const(instance) + return llmemory.weakref_create(llinstance) Modified: pypy/dist/pypy/rpython/test/test_rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rweakref.py (original) +++ pypy/dist/pypy/rpython/test/test_rweakref.py Tue Sep 11 12:17:49 2007 @@ -1,4 +1,4 @@ -import weakref +import py, weakref from pypy.rlib import rgc from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.test.test_llinterp import interpret @@ -39,9 +39,52 @@ x.hello = 64 r = weakref.ref(x) return r().hello, x # returns 'x' too, to keep it alive - res = interpret(f, [1]) assert res.item0 == 42 - res = interpret(f, [0]) assert res.item0 == 64 + +def test_prebuilt_weakref(): + class A: + pass + a1 = A() + a1.hello = 5 + w1 = weakref.ref(a1) + a2 = A() + a2.hello = 8 + w2 = weakref.ref(a2) + + def f(n): + if n: + r = w1 + else: + r = w2 + return r().hello + res = interpret(f, [1]) + assert res == 5 + res = interpret(f, [0]) + assert res == 8 + +def test_prebuilt_dead_weakref(): + py.test.skip("in-progress") + class A: + pass + a1 = A() + w1 = weakref.ref(a1) + a2 = A() + w2 = weakref.ref(a2) + + del a1 + rgc.collect() + assert w1() is None + + def f(n): + if n: + r = w1 + else: + r = w2 + return r() is not None + res = interpret(f, [1]) + assert res == False + res = interpret(f, [0]) + assert res == True From arigo at codespeak.net Tue Sep 11 12:23:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 12:23:42 +0200 (CEST) Subject: [pypy-svn] r46467 - in pypy/dist/pypy: annotation rpython/test Message-ID: <20070911102342.03CE08101@code0.codespeak.net> Author: arigo Date: Tue Sep 11 12:23:42 2007 New Revision: 46467 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/rpython/test/test_rweakref.py Log: Prebuilt weakrefs: changes missing from the previous check-in, together with support for dead weakrefs. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Tue Sep 11 12:23:42 2007 @@ -11,7 +11,8 @@ from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator from pypy.annotation.model import SomePBC, SomeSlice, SomeFloat, s_None -from pypy.annotation.model import SomeExternalObject, SomeWeakRef +from pypy.annotation.model import SomeExternalObject +from pypy.annotation.model import SomeWeakRef, SomeDeadWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeWeakGcAddress from pypy.annotation.model import SomeCTypesObject @@ -876,6 +877,18 @@ return SomeObject() return SomeWeakRef(basedef) +class __extend__(pairtype(SomeWeakRef, SomeDeadWeakRef)): + def union((s_wrf1, s_wrf2)): + return SomeWeakRef(s_wrf1.classdef) + +class __extend__(pairtype(SomeDeadWeakRef, SomeWeakRef)): + def union((s_wrf1, s_wrf2)): + return SomeWeakRef(s_wrf2.classdef) + +class __extend__(pairtype(SomeDeadWeakRef, SomeDeadWeakRef)): + def union((s_wrf1, s_wrf2)): + return SomeDeadWeakRef() + #_________________________________________ # memory addresses Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Tue Sep 11 12:23:42 2007 @@ -3,7 +3,7 @@ """ from __future__ import generators -import sys, types, inspect +import sys, types, inspect, weakref from pypy.objspace.flow.model import Constant from pypy.annotation.model import SomeString, SomeChar, SomeFloat, \ @@ -11,7 +11,8 @@ SomeInteger, SomeExternalObject, SomeOOInstance, TLS, SomeAddress, \ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ - SomeList, SomeObject, SomeWeakGcAddress, HarmlesslyBlocked + SomeList, SomeObject, SomeWeakGcAddress, HarmlesslyBlocked, \ + SomeWeakRef, SomeDeadWeakRef from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -388,6 +389,14 @@ dictdef.generalize_key(self.immutablevalue(ek, False)) dictdef.generalize_value(self.immutablevalue(ev, False)) result = SomeDict(dictdef) + elif tp is weakref.ref: + x1 = x() + if x1 is None: + result = SomeDeadWeakRef() + else: + s1 = self.immutablevalue(x1) + assert isinstance(s1, SomeInstance) + result = SomeWeakRef(s1.classdef) elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") result = SomeBuiltin(BUILTIN_ANALYZERS[x], methodname="%s.%s" % (_module, x.__name__)) Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Tue Sep 11 12:23:42 2007 @@ -503,6 +503,13 @@ def can_be_none(self): return False +class SomeDeadWeakRef(SomeObject): + knowntype = weakref.ref + immutable = True + + def can_be_none(self): + return False + class SomeLLWeakRef(SomeObject): immutable = True Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Tue Sep 11 12:23:42 2007 @@ -7,9 +7,9 @@ SomeDict, SomeUnicodeCodePoint, SomeTuple, SomeImpossibleValue, \ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ - SomeCTypesObject, SomeWeakRef, s_ImpossibleValue, s_Bool, \ + SomeCTypesObject, s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ - SomeGenericCallable + SomeGenericCallable, SomeWeakRef, SomeDeadWeakRef from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? @@ -757,6 +757,10 @@ def simple_call(s_wrf): return SomeInstance(s_wrf.classdef, can_be_None=True) +class __extend__(SomeDeadWeakRef): + def simple_call(s_wrf): + return s_None + #_________________________________________ # memory addresses Modified: pypy/dist/pypy/rpython/test/test_rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rweakref.py (original) +++ pypy/dist/pypy/rpython/test/test_rweakref.py Tue Sep 11 12:23:42 2007 @@ -66,7 +66,6 @@ assert res == 8 def test_prebuilt_dead_weakref(): - py.test.skip("in-progress") class A: pass a1 = A() From antocuni at codespeak.net Tue Sep 11 14:15:37 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 11 Sep 2007 14:15:37 +0200 (CEST) Subject: [pypy-svn] r46468 - in pypy/dist/pypy/translator/jvm: . src/pypy Message-ID: <20070911121537.B942B8103@code0.codespeak.net> Author: antocuni Date: Tue Sep 11 14:15:36 2007 New Revision: 46468 Added: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/typesystem.py Log: move ll_os_* to their own class. Wrap std{out,in,err} into file descriptors. Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Tue Sep 11 14:15:36 2007 @@ -10,7 +10,7 @@ jObject, jByteArray, jPyPyExcWrap, jIntegerClass, jLongClass, \ jDoubleClass, jCharClass, jStringBuilder, JvmScalarType, jArrayList, \ jObjectArray, jPyPyInterlink, jPyPyCustomDict, jPyPyEquals, \ - jPyPyHashCode, jMap, jWeakRef, jSystem + jPyPyHashCode, jMap, jWeakRef, jSystem, jll_os # ___________________________________________________________________________ # Miscellaneous helper functions @@ -1020,7 +1020,11 @@ def call_primitive(self, op, module, name): callee = op.args[0].value argtypes, rettype = self.db.types_for_signature(callee._TYPE.ARGS, callee._TYPE.RESULT) - mthd = Method.s(jPyPy, name, argtypes, rettype) + if module == 'll_os': + jcls = jll_os + else: + jcls = jPyPy + mthd = Method.s(jcls, name, argtypes, rettype) self.emit(mthd) def call_oostring(self, OOTYPE): Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Tue Sep 11 14:15:36 2007 @@ -176,6 +176,7 @@ 'Constants', 'StatResult', 'PyPy', + 'll_os', )) def _make_str(self, a): Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Tue Sep 11 14:15:36 2007 @@ -792,65 +792,6 @@ return System.currentTimeMillis()/1000.0; } - public static int ll_os_write(int fd, String text) { - // TODO: file descriptors, etc - if (fd == 1) - System.out.print(text); - else if (fd == 2) - System.err.print(text); - else - throw new RuntimeException("Invalid FD"); - return text.length(); - } - - public static ArrayList ll_os_envitems() - { - return new ArrayList(); // XXX - } - - public static String ll_os_getcwd() - { - return System.getProperty("user.dir"); - } - - public static StatResult ll_os_stat(String path) - { - if (path.equals("")) - throwOSError(ENOENT, "No such file or directory: ''"); - - File f = new File(path); - - if (f.exists()) { - StatResult res = new StatResult(); - if (f.isDirectory()) - res.setMode(S_IFDIR); - else { - res.setMode(S_IFREG); - res.setSize(f.length()); - res.setMtime((int)f.lastModified()); - } - return res; - } - - throwOSError(ENOENT, "No such file or directory: '"+path+"'"); - return null; // never reached - } - - public static int ll_os_open(String path, int flags, int mode) - { - throwOSError(ENOENT, "DUMMY: No such file or directory: '"+path+"'"); // XXX - return -1; // never reached - } - - public static StatResult ll_os_lstat(String path) - { - return ll_os_stat(path); // XXX - } - - public static String ll_os_strerror(int errno) - { - return "errno: " + errno; - } public static String ll_join(String a, String b) { @@ -980,10 +921,6 @@ public static void throwValueError() { interlink.throwValueError(); } - - public static void throwOSError(int errCode, String errText) { - interlink.throwOSError(errCode); // errText currently ignored... fix? - } // ---------------------------------------------------------------------- // Self Test Added: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Tue Sep 11 14:15:36 2007 @@ -0,0 +1,157 @@ +package pypy; + +import java.io.*; +import java.util.HashMap; +import java.util.ArrayList; +import java.util.Map; + +abstract class FileWrapper +{ + public abstract void write(String buffer); + public abstract String read(int count); +} + +class PrintStreamWrapper extends FileWrapper +{ + private PrintStream stream; + + public PrintStreamWrapper(PrintStream stream) + { + this.stream = stream; + } + + public void write(String buffer) + { + this.stream.print(buffer); + } + + public String read(int count) + { + ll_os.throwOSError(PyPy.EBADF, "Write-only fd"); + return null; // never reached + } +} + +class InputStreamWrapper extends FileWrapper +{ + private InputStream stream; + + public InputStreamWrapper(InputStream stream) + { + this.stream = stream; + } + + public void write(String buffer) + { + ll_os.throwOSError(PyPy.EBADF, "Read-only fd"); + } + + public String read(int count) + { + try { + byte[] buf = new byte[count]; + int n = stream.read(buf, 0, count); + return new String(buf); + } + catch(IOException e) { + ll_os.throwOSError(PyPy.EIO, e.getMessage()); + return null; // never reached + } + } +} + + +public class ll_os { + + // NB: these values are those used by Windows and they differs + // from the Unix ones; the os module is patched with these + // values before flowgraphing to make sure we get the very + // same values on each platform we do the compilation. + private static final int O_RDONLY = 0x0000; + private static final int O_WRONLY = 0x0001; + private static final int O_RDWR = 0x0002; + private static final int O_APPEND = 0x0008; + private static final int O_CREAT = 0x0100; + private static final int O_TRUNC = 0x0200; + private static final int O_TEXT = 0x4000; + private static final int O_BINARY = 0x8000; + + private static final int S_IFMT = 61440; + private static final int S_IFDIR = 16384; + private static final int S_IFREG = 32768; + + private static Map FileDescriptors = new HashMap(); + private static Map ErrorMessages = new HashMap(); + + static { + FileDescriptors.put(new Integer(0), new PrintStreamWrapper(System.out)); + FileDescriptors.put(new Integer(1), new InputStreamWrapper(System.in)); + FileDescriptors.put(new Integer(2), new PrintStreamWrapper(System.err)); + } + + public static void throwOSError(int errno, String errText) { + ErrorMessages.put(new Integer(errno), errText); + PyPy.interlink.throwOSError(errno); + } + + public static int ll_os_open(String name, int flags, int mode) + { + throwOSError(PyPy.ENOENT, "DUMMY ll_os_open"); + return -1; + } + + public static StatResult ll_os_lstat(String path) + { + return ll_os_stat(path); // XXX + } + + public static String ll_os_strerror(int errno) + { + String msg = ErrorMessages.remove(new Integer(errno)); + if (msg == null) + return "errno: " + errno; + else + return msg; + } + + public static int ll_os_write(int fd, String text) { + FileWrapper f = FileDescriptors.get(new Integer(fd)); + if (f == null) + throwOSError(PyPy.EBADF, "Invalid fd: " + fd); + f.write(text); + return text.length(); + } + + public static ArrayList ll_os_envitems() + { + return new ArrayList(); // XXX + } + + public static String ll_os_getcwd() + { + return System.getProperty("user.dir"); + } + + public static StatResult ll_os_stat(String path) + { + if (path.equals("")) + ll_os.throwOSError(PyPy.ENOENT, "No such file or directory: ''"); + + File f = new File(path); + + if (f.exists()) { + StatResult res = new StatResult(); + if (f.isDirectory()) + res.setMode(S_IFDIR); + else { + res.setMode(S_IFREG); + res.setSize(f.length()); + res.setMtime((int)f.lastModified()); + } + return res; + } + + ll_os.throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); + return null; // never reached + } +} Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Tue Sep 11 14:15:36 2007 @@ -178,6 +178,7 @@ jPyPyInterlink = JvmClassType('pypy.Interlink') jPyPyCustomDict = JvmClassType('pypy.CustomDict') jPyPyStatResult = JvmClassType('pypy.StatResult') +jll_os = JvmClassType('pypy.ll_os') jArithmeticException = JvmClassType('java.lang.ArithmeticException', throwable=True) From arigo at codespeak.net Tue Sep 11 14:50:36 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 14:50:36 +0200 (CEST) Subject: [pypy-svn] r46469 - in pypy/dist/pypy: rpython rpython/lltypesystem rpython/memory/gctransform translator translator/c translator/c/src translator/c/test Message-ID: <20070911125036.45A1080FB@code0.codespeak.net> Author: arigo Date: Tue Sep 11 14:50:34 2007 New Revision: 46469 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/memory/gctransform/boehm.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/primitive.py pypy/dist/pypy/translator/c/src/mem.h pypy/dist/pypy/translator/c/test/test_boehm.py pypy/dist/pypy/translator/exceptiontransform.py Log: Weakrefs for genc with Boehm. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Tue Sep 11 14:50:34 2007 @@ -789,6 +789,9 @@ def op_boehm_register_finalizer(self, p, finalizer): pass + def op_boehm_disappearing_link(self, link, obj): + pass + def op_raw_malloc_usage(self, size): assert lltype.typeOf(size) == lltype.Signed return self.heap.raw_malloc_usage(size) Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Tue Sep 11 14:50:34 2007 @@ -489,6 +489,14 @@ # ____________________________________________________________ +# +# WeakRef - like in RPython, they are pointers that don't keep their +# target malloc'ed structure alive. When the target dies +# all WeakRefs to it are cleared. +# +# WeakGcAddress - like WeakRef but not automatically cleared. There is +# no direct way to know if the target is still alive. +# We should more or less deprecate them in favor of WeakRef. class fakeweakaddress(object): # XXX convoluted code to support both lltype._ptr and simulatorptr Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Tue Sep 11 14:50:34 2007 @@ -339,6 +339,7 @@ 'boehm_malloc': LLOp(), 'boehm_malloc_atomic': LLOp(), 'boehm_register_finalizer': LLOp(), + 'boehm_disappearing_link': LLOp(), 'raw_malloc': LLOp(), 'raw_malloc_usage': LLOp(sideeffects=False), 'raw_free': LLOp(), @@ -366,8 +367,6 @@ 'cast_weakadr_to_int': LLOp(canfold=True), 'cast_adr_to_int': LLOp(canfold=True), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp - 'weakref_create': LLOp(sideeffects=False), - 'weakref_deref': LLOp(sideeffects=False), # __________ used by the JIT ________ @@ -401,6 +400,13 @@ # NOTE NOTE NOTE! don't forget *** canunwindgc=True *** for anything that # can go through a stack unwind, in particular anything that mallocs! + # __________ weakrefs __________ + + # weakref_create can raise MemoryError in GCs like Boehm, but not + # in the framework GC, so it should never cause a stack unwind. + 'weakref_create': LLOp(canraise=(MemoryError,), sideeffects=False), + 'weakref_deref': LLOp(sideeffects=False), + # __________ stackless operation(s) __________ 'yield_current_frame_to_caller': LLOp(canraise=(StackException,)), Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Tue Sep 11 14:50:34 2007 @@ -42,6 +42,11 @@ ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address, inline=False) self.malloc_varsize_ptr = self.inittime_helper( ll_malloc_varsize, [lltype.Signed]*4, llmemory.Address, inline=False) + self.weakref_create_ptr = self.inittime_helper( + ll_weakref_create, [llmemory.Address], llmemory.WeakRef, + inline=False) + self.weakref_deref_ptr = self.inittime_helper( + ll_weakref_deref, [llmemory.WeakRef], llmemory.Address) self.mixlevelannotator.finish() # for now self.mixlevelannotator.backend_optimize() @@ -126,3 +131,43 @@ self.finalizer_funcptrs[TYPE] = fptr return fptr + def gct_weakref_create(self, hop): + v_instance, = hop.spaceop.args + v_addr = hop.genop("cast_ptr_to_adr", [v_instance], + resulttype=llmemory.Address) + v_wref = hop.genop("direct_call", + [self.weakref_create_ptr, v_addr], + resulttype=llmemory.WeakRef) + hop.cast_result(v_wref) + + def gct_weakref_deref(self, hop): + v_wref, = hop.spaceop.args + v_addr = hop.genop("direct_call", + [self.weakref_deref_ptr, v_wref], + resulttype=llmemory.Address) + hop.cast_result(v_addr) + + +########## weakrefs ########## +# Boehm: we implement weakrefs with an extra indirection: GCWeakRef is a +# pointer to malloced data containing only a Boehm disappearing link. +# This allows the disappearing link to remain at a fixed address. +# We also don't have to hide the link's value with HIDE_POINTER(), +# because we use GC_MALLOC_ATOMIC(). + +# Declared in gc.py: typedef GC_PTR *GCWeakRef; + +WEAKLINK = lltype.FixedSizeArray(llmemory.Address, 1) +sizeof_weakreflink = llmemory.sizeof(WEAKLINK) + +def ll_weakref_create(targetaddr): + link = llop.boehm_malloc_atomic(llmemory.Address, sizeof_weakreflink) + link.address[0] = targetaddr + llop.boehm_disappearing_link(lltype.Void, link, targetaddr) + # abuse of llop.cast_pointer() + return llop.cast_pointer(llmemory.WeakRef, link) + +def ll_weakref_deref(wref): + # abuse of llop.cast_pointer() + link = llop.cast_pointer(llmemory.Address, wref) + return link and link.address[0] Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Tue Sep 11 14:50:34 2007 @@ -204,7 +204,7 @@ yield '#define USING_BOEHM_GC' def pre_gc_code(self): - return [] + return ['typedef GC_PTR *GCWeakRef;'] def gc_startup_code(self): if sys.platform == 'win32': Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Tue Sep 11 14:50:34 2007 @@ -6,7 +6,7 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, WeakGcAddress, fakeweakaddress, \ - GCHeaderOffset + GCHeaderOffset, WeakRef from pypy.translator.c.support import cdecl # ____________________________________________________________ @@ -149,6 +149,7 @@ Void: 'void @', Address: 'void* @', WeakGcAddress: 'GC_hidden_pointer @', + WeakRef: 'GCWeakRef @', } PrimitiveErrorValue = { @@ -163,6 +164,7 @@ Void: '/* error */', Address: 'NULL', WeakGcAddress: 'HIDE_POINTER(NULL)', + WeakRef: 'NULL', } def define_c_primitive(ll_type, c_name): Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Tue Sep 11 14:50:34 2007 @@ -100,6 +100,9 @@ */ #define OP_CALL_BOEHM_GC_ALLOC(size, r) OP_BOEHM_ZERO_MALLOC(size, r, void *, 0, 0) +#define OP_BOEHM_DISAPPEARING_LINK(link, obj, r) \ + GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj) + #endif /* USING_BOEHM_GC */ /************************************************************/ Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Tue Sep 11 14:50:34 2007 @@ -284,6 +284,40 @@ res = c_fn() assert res == 2 + def test_weakref(self): + import weakref + from pypy.rlib import rgc + + class A: + pass + + def fn(n): + keepalive = [] + weakrefs = [] + a = None + for i in range(n): + if i & 1 == 0: + a = A() + a.index = i + weakrefs.append(weakref.ref(a)) + if i % 7 == 6: + keepalive.append(a) + rgc.collect() + count_free = 0 + for i in range(n): + a = weakrefs[i]() + if i % 7 == 6: + assert a is not None + if a is not None: + assert a.index == i & ~1 + else: + count_free += 1 + return count_free + c_fn = self.getcompiled(fn, [int]) + res = c_fn(7000) + # more than half of them should have been freed, ideally up to 6000 + assert 3500 <= res <= 6000 + class TestUsingExactBoehm(TestUsingBoehm): gcpolicy = "exact_boehm" Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Tue Sep 11 14:50:34 2007 @@ -25,6 +25,7 @@ lltype.Bool: True, llmemory.Address: NULL, llmemory.WeakGcAddress: llmemory.fakeweakaddress(None), + llmemory.WeakRef: llmemory.fakeweakref(None), lltype.Void: None} def error_value(T): From arigo at codespeak.net Tue Sep 11 15:19:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 15:19:51 +0200 (CEST) Subject: [pypy-svn] r46470 - in pypy/dist/pypy: rpython/memory/gctransform translator/c translator/c/test Message-ID: <20070911131951.128AE8102@code0.codespeak.net> Author: arigo Date: Tue Sep 11 15:19:50 2007 New Revision: 46470 Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/primitive.py pypy/dist/pypy/translator/c/test/test_boehm.py Log: Prebuilt weakrefs in genc/Boehm. Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Tue Sep 11 15:19:50 2007 @@ -171,3 +171,12 @@ # abuse of llop.cast_pointer() link = llop.cast_pointer(llmemory.Address, wref) return link and link.address[0] + +def convert_prebuilt_weakref_to(targetptr): + # Prebuilt weakrefs don't really need to be weak at all, + # but we need to emulate the structure expected by ll_weakref_deref(). + # This is essentially the same code as in ll_weakref_create(), but I'm + # not sure trying to share it is worth the hassle... + link = lltype.malloc(WEAKLINK, immortal=True) + link[0] = llmemory.cast_ptr_to_adr(targetptr) + return link Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Tue Sep 11 15:19:50 2007 @@ -175,6 +175,10 @@ class BoehmGcPolicy(BasicGcPolicy): transformerclass = boehm.BoehmGCTransformer + def __init__(self, *args, **kwds): + BasicGcPolicy.__init__(self, *args, **kwds) + self.weakref_llwrapper_cache = {} + def array_setup(self, arraydefnode): pass @@ -213,6 +217,16 @@ yield 'GC_all_interior_pointers = 0;' yield 'GC_init();' + def name_weakref_to(self, target): + # the cache is essential to ensure that repeated calls to + # db.get(weakref) don't return new llwrapper structures all + # the time, which defeats the db.complete() logic. + try: + llwrapper = self.weakref_llwrapper_cache[target._obj] + except KeyError: + llwrapper = boehm.convert_prebuilt_weakref_to(target) + self.weakref_llwrapper_cache[target._obj] = llwrapper + return '((GCWeakRef)%s)' % (self.db.get(llwrapper),) def OP_GC_FETCH_EXCEPTION(self, funcgen, op): result = funcgen.expr(op.result) Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Tue Sep 11 15:19:50 2007 @@ -6,7 +6,7 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, WeakGcAddress, fakeweakaddress, \ - GCHeaderOffset, WeakRef + GCHeaderOffset, WeakRef, fakeweakref from pypy.translator.c.support import cdecl # ____________________________________________________________ @@ -121,6 +121,14 @@ assert ob is not None return 'HIDE_POINTER(%s)'%db.get(ob) +def name_weakref(value, db): + assert isinstance(value, fakeweakref) + target = value.get() + if target is None: + return 'NULL' + else: + return db.gcpolicy.name_weakref_to(target) + # On 64 bit machines, SignedLongLong and Signed are the same, so the # order matters, because we want the Signed implementation. PrimitiveName = { @@ -135,6 +143,7 @@ Void: name_void, Address: name_address, WeakGcAddress: name_weakgcaddress, + WeakRef: name_weakref, } PrimitiveType = { Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Tue Sep 11 15:19:50 2007 @@ -318,6 +318,33 @@ # more than half of them should have been freed, ideally up to 6000 assert 3500 <= res <= 6000 + def test_prebuilt_weakref(self): + import weakref + from pypy.rlib import rgc + class A: + pass + a = A() + a.hello = 42 + r1 = weakref.ref(a) + r2 = weakref.ref(A()) + rgc.collect() + assert r2() is None + def fn(n): + if n: + r = r1 + else: + r = r2 + a = r() + if a is None: + return -5 + else: + return a.hello + c_fn = self.getcompiled(fn, [int]) + res = c_fn(1) + assert res == 42 + res = c_fn(0) + assert res == -5 + class TestUsingExactBoehm(TestUsingBoehm): gcpolicy = "exact_boehm" From arigo at codespeak.net Tue Sep 11 15:42:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 15:42:51 +0200 (CEST) Subject: [pypy-svn] r46471 - in pypy/dist/pypy: annotation rpython Message-ID: <20070911134251.85CC6811B@code0.codespeak.net> Author: arigo Date: Tue Sep 11 15:42:51 2007 New Revision: 46471 Modified: pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/rpython/rweakref.py Log: Vague attempt at Python 2.3 compatibility. Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Tue Sep 11 15:42:51 2007 @@ -389,7 +389,7 @@ dictdef.generalize_key(self.immutablevalue(ek, False)) dictdef.generalize_value(self.immutablevalue(ev, False)) result = SomeDict(dictdef) - elif tp is weakref.ref: + elif tp is weakref.ReferenceType: x1 = x() if x1 is None: result = SomeDeadWeakRef() Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Tue Sep 11 15:42:51 2007 @@ -495,7 +495,7 @@ # weakrefs class SomeWeakRef(SomeObject): - knowntype = weakref.ref + knowntype = weakref.ReferenceType immutable = True def __init__(self, classdef): self.classdef = classdef @@ -504,7 +504,7 @@ return False class SomeDeadWeakRef(SomeObject): - knowntype = weakref.ref + knowntype = weakref.ReferenceType immutable = True def can_be_none(self): Modified: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/rweakref.py (original) +++ pypy/dist/pypy/rpython/rweakref.py Tue Sep 11 15:42:51 2007 @@ -38,7 +38,7 @@ return hop.genop('weakref_deref', [v_wref], resulttype=hop.r_result) def convert_const(self, value): - assert isinstance(value, weakref.ref) + assert isinstance(value, weakref.ReferenceType) instance = value() bk = self.rtyper.annotator.bookkeeper # obscure! if the annotator hasn't seen this object before, From arigo at codespeak.net Tue Sep 11 16:00:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 16:00:01 +0200 (CEST) Subject: [pypy-svn] r46472 - in pypy/dist/pypy: config doc/config rpython Message-ID: <20070911140001.DE3698103@code0.codespeak.net> Author: arigo Date: Tue Sep 11 16:00:00 2007 New Revision: 46472 Added: pypy/dist/pypy/doc/config/translation.rweakref.txt (contents, props changed) Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/rpython/rweakref.py Log: Added a configuration "option", translator.rweakref, set to False when the backend or GC policy doesn't support RPython-level weakrefs. Can be tested in an RPython program to select between two implementation strategies. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Tue Sep 11 16:00:00 2007 @@ -20,7 +20,8 @@ "ootype": [("translation.backendopt.raisingop2direct_call", False), ("translation.backendopt.constfold", False), ("translation.backendopt.heap2stack", False), - ("translation.backendopt.clever_malloc_removal", False)] + ("translation.backendopt.clever_malloc_removal", False), + ("translation.rweakref", False)], # XXX }), ChoiceOption("backend", "Backend to use for code generation", ["c", "llvm", "cli", "jvm", "js", "squeak", "cl"], @@ -43,7 +44,11 @@ ["boehm", "ref", "framework", "none", "stacklessgc", "exact_boehm"], "ref", requires={ - "stacklessgc": [("translation.stackless", True)]}, + "ref": [("translation.rweakref", False)], # XXX + "framework": [("translation.rweakref", False)], # XXX + "none": [("translation.rweakref", False)], # XXX + "stacklessgc": [("translation.stackless", True), + ("translation.rweakref", False)]}, # XXX cmdline="--gc"), BoolOption("thread", "enable use of threading primitives", default=False, cmdline="--thread", @@ -60,6 +65,8 @@ cmdline=None), BoolOption("sandbox", "Produce a fully-sandboxed executable", default=False, cmdline="--sandbox"), + BoolOption("rweakref", "The backend supports RPython-level weakrefs", + default=True), # misc StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"), Added: pypy/dist/pypy/doc/config/translation.rweakref.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/translation.rweakref.txt Tue Sep 11 16:00:00 2007 @@ -0,0 +1,3 @@ +This indicates if the backend and GC policy support RPython-level weakrefs. +Can be tested in an RPython program to select between two implementation +strategies. Modified: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/rweakref.py (original) +++ pypy/dist/pypy/rpython/rweakref.py Tue Sep 11 16:00:00 2007 @@ -1,6 +1,7 @@ import weakref from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Constant +from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr from pypy.rpython.rclass import getinstancerepr from pypy.rpython.lltypesystem import lltype, llmemory @@ -31,6 +32,9 @@ def __init__(self, rtyper): self.rtyper = rtyper + if not rtyper.getconfig().translation.rweakref: + raise TyperError("RPython-level weakrefs are not supported by " + "this backend or GC policy") def rtype_simple_call(self, hop): v_wref, = hop.inputargs(self) From arigo at codespeak.net Tue Sep 11 17:07:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 17:07:01 +0200 (CEST) Subject: [pypy-svn] r46473 - pypy/dist/pypy/objspace/std Message-ID: <20070911150701.20E2B80F8@code0.codespeak.net> Author: arigo Date: Tue Sep 11 17:07:00 2007 New Revision: 46473 Modified: pypy/dist/pypy/objspace/std/typeobject.py pypy/dist/pypy/objspace/std/typetype.py Log: Use RPython-level weakrefs when available to implement __subclasses__() and assignment to __bases__. When they are not available, we simply disable these two features for now. Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Tue Sep 11 17:07:00 2007 @@ -59,7 +59,7 @@ w_self.nslots = 0 w_self.needsdel = False w_self.w_bestbase = None - w_self.weak_subclasses_w = [] + w_self.weak_subclasses = [] # make sure there is a __doc__ in dict_w if '__doc__' not in dict_w: @@ -403,43 +403,40 @@ def add_subclass(w_self, w_subclass): space = w_self.space - if space.config.translation.sandbox: - return # XXX weakrefs are disabled in a sandbox translation ATM - from pypy.module._weakref.interp__weakref import basic_weakref - w_newref = basic_weakref(space, w_subclass) - - for i in range(len(w_self.weak_subclasses_w)): - w_ref = w_self.weak_subclasses_w[i] - ob = space.call_function(w_ref) - if space.is_w(ob, space.w_None): - w_self.weak_subclasses_w[i] = w_newref + if not space.config.translation.rweakref: + return # no weakref support, don't keep track of subclasses + import weakref + assert isinstance(w_subclass, W_TypeObject) + newref = weakref.ref(w_subclass) + for i in range(len(w_self.weak_subclasses)): + ref = w_self.weak_subclasses[i] + if ref() is None: + w_self.weak_subclasses[i] = newref return else: - w_self.weak_subclasses_w.append(w_newref) + w_self.weak_subclasses.append(newref) def remove_subclass(w_self, w_subclass): space = w_self.space - if space.config.translation.sandbox: - return # XXX weakrefs are disabled in a sandbox translation ATM - - for i in range(len(w_self.weak_subclasses_w)): - w_ref = w_self.weak_subclasses_w[i] - ob = space.call_function(w_ref) - if space.is_w(ob, w_subclass): - del w_self.weak_subclasses_w[i] + if not space.config.translation.rweakref: + return # no weakref support, don't keep track of subclasses + for i in range(len(w_self.weak_subclasses)): + ref = w_self.weak_subclasses[i] + if ref() is w_subclass: + del w_self.weak_subclasses[i] return def get_subclasses(w_self): space = w_self.space - if space.config.translation.sandbox: - msg = ("weakrefs are disabled in a sandbox translation " - "at the moment") + if not space.config.translation.rweakref: + msg = ("this feature requires weakrefs, " + "which are not available in this build of PyPy") raise OperationError(space.w_RuntimeError, space.wrap(msg)) subclasses_w = [] - for w_ref in w_self.weak_subclasses_w: - w_ob = space.call_function(w_ref) - if not space.is_w(w_ob, space.w_None): + for ref in w_self.weak_subclasses: + w_ob = ref() + if w_ob is not None: subclasses_w.append(w_ob) return subclasses_w Modified: pypy/dist/pypy/objspace/std/typetype.py ============================================================================== --- pypy/dist/pypy/objspace/std/typetype.py (original) +++ pypy/dist/pypy/objspace/std/typetype.py Tue Sep 11 17:07:00 2007 @@ -92,15 +92,11 @@ def mro_subclasses(space, w_type, temp): from pypy.objspace.std.typeobject import W_TypeObject - if not w_type.weak_subclasses_w: - return - for w_ref in w_type.weak_subclasses_w: - w_sc = space.call_function(w_ref) - if not space.is_w(w_sc, space.w_None): - assert isinstance(w_sc, W_TypeObject) - temp.append((w_sc, w_sc.mro_w)) - mro_internal(space, w_sc) - mro_subclasses(space, w_sc, temp) + for w_sc in w_type.get_subclasses(): + assert isinstance(w_sc, W_TypeObject) + temp.append((w_sc, w_sc.mro_w)) + mro_internal(space, w_sc) + mro_subclasses(space, w_sc, temp) # should be a W_TypeObject method i guess def mro_internal(space, w_type): From cfbolz at codespeak.net Tue Sep 11 17:26:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Sep 2007 17:26:45 +0200 (CEST) Subject: [pypy-svn] r46474 - pypy/dist/pypy/module/_weakref Message-ID: <20070911152645.4F7968101@code0.codespeak.net> Author: cfbolz Date: Tue Sep 11 17:26:44 2007 New Revision: 46474 Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py Log: rewrite the _weakref module to use the fancy new rweakref support. Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Tue Sep 11 17:26:44 2007 @@ -4,96 +4,83 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.gateway import interp2app, ObjSpace -from pypy.rlib.objectmodel import cast_weakgcaddress_to_object, cast_object_to_weakgcaddress from pypy.rpython.lltypesystem.llmemory import WEAKNULL +import weakref class WeakrefLifeline(object): def __init__(self): - self.addr_refs = [] + self.refs_weak = [] self.cached_weakref_index = -1 self.cached_proxy_index = -1 def __del__(self): - for i in range(len(self.addr_refs) - 1, -1, -1): - addr_ref = self.addr_refs[i] - w_ref = cast_weakgcaddress_to_object(addr_ref, W_WeakrefBase) - if w_ref is not None: - w_ref.invalidate() - for i in range(len(self.addr_refs) - 1, -1, -1): - addr_ref = self.addr_refs[i] - w_ref = cast_weakgcaddress_to_object(addr_ref, W_WeakrefBase) + for i in range(len(self.refs_weak) - 1, -1, -1): + w_ref = self.refs_weak[i]() if w_ref is not None: w_ref.activate_callback() - def get_weakref(self, space, w_subtype, w_obj, w_callable): + def get_or_make_weakref(self, space, w_subtype, w_obj, w_callable): w_weakreftype = space.gettypeobject(W_Weakref.typedef) is_weakreftype = space.is_w(w_weakreftype, w_subtype) can_reuse = space.is_w(w_callable, space.w_None) if is_weakreftype and can_reuse and self.cached_weakref_index >= 0: - cached_weakref_address = self.addr_refs[self.cached_weakref_index] - return cast_weakgcaddress_to_object(cached_weakref_address, W_Weakref) + w_cached = self.refs_weak[self.cached_weakref_index]() + if w_cached is not None: + return w_cached + else: + self.cached_weakref_index = -1 w_ref = space.allocate_instance(W_Weakref, w_subtype) - index = len(self.addr_refs) - W_Weakref.__init__(w_ref, space, self, index, - w_obj, w_callable) - self.addr_refs.append(cast_object_to_weakgcaddress(w_ref)) + index = len(self.refs_weak) + W_Weakref.__init__(w_ref, space, w_obj, w_callable) + self.refs_weak.append(weakref.ref(w_ref)) if is_weakreftype and can_reuse: self.cached_weakref_index = index return w_ref - def get_proxy(self, space, w_obj, w_callable): + def get_or_make_proxy(self, space, w_obj, w_callable): can_reuse = space.is_w(w_callable, space.w_None) if can_reuse and self.cached_proxy_index >= 0: - cached_proxy_address = self.addr_refs[self.cached_proxy_index] - return cast_weakgcaddress_to_object(cached_proxy_address, W_Proxy) - index = len(self.addr_refs) + w_cached = self.refs_weak[self.cached_proxy_index]() + if w_cached is not None: + return w_cached + else: + self.cached_proxy_index = -1 + index = len(self.refs_weak) if space.is_true(space.callable(w_obj)): - w_proxy = W_CallableProxy(space, self, index, w_obj, w_callable) + w_proxy = W_CallableProxy(space, w_obj, w_callable) else: - w_proxy = W_Proxy(space, self, index, w_obj, w_callable) - self.addr_refs.append(cast_object_to_weakgcaddress(w_proxy)) + w_proxy = W_Proxy(space, w_obj, w_callable) + self.refs_weak.append(weakref.ref(w_proxy)) if can_reuse: self.cached_proxy_index = index return w_proxy - def ref_is_dead(self, index): - self.addr_refs[index] = WEAKNULL - if self.cached_proxy_index == index: - self.cached_proxy_index = -1 - if self.cached_weakref_index == index: - self.cached_weakref_index = -1 - def get_any_weakref(self, space): if self.cached_weakref_index != -1: - return cast_weakgcaddress_to_object( - self.addr_refs[self.cached_weakref_index], W_Root) + w_ref = self.refs_weak[self.cached_weakref_index]() + if w_ref is not None: + return w_ref w_weakreftype = space.gettypeobject(W_Weakref.typedef) - for i in range(len(self.addr_refs)): - addr = self.addr_refs[i] - if cast_weakgcaddress_to_object(addr, W_Root) is not None: - w_ref = cast_weakgcaddress_to_object(addr, W_Root) - if space.is_true(space.isinstance(w_ref, w_weakreftype)): - return w_ref + for i in range(len(self.refs_weak)): + w_ref = self.refs_weak[i]() + if (w_ref is not None and + space.is_true(space.isinstance(w_ref, w_weakreftype))): + return w_ref return space.w_None class W_WeakrefBase(Wrappable): - def __init__(w_self, space, lifeline, index, w_obj, w_callable): + def __init__(w_self, space, w_obj, w_callable): w_self.space = space - w_self.address = cast_object_to_weakgcaddress(w_obj) + w_self.w_obj_weak = weakref.ref(w_obj) w_self.w_callable = w_callable - w_self.addr_lifeline = cast_object_to_weakgcaddress(lifeline) - w_self.index = index def dereference(self): - if cast_weakgcaddress_to_object(self.address, W_Root) is None: + w_obj = self.w_obj_weak() + if w_obj is None: return self.space.w_None - return cast_weakgcaddress_to_object(self.address, W_Root) + return w_obj - def invalidate(w_self): - w_self.address = WEAKNULL - w_self.addr_lifeline = WEAKNULL - def activate_callback(w_self): if not w_self.space.is_w(w_self.w_callable, w_self.space.w_None): try: @@ -101,15 +88,10 @@ except OperationError, e: e.write_unraisable(w_self.space, 'function', w_self.w_callable) - def __del__(w_self): - lifeline = cast_weakgcaddress_to_object(w_self.addr_lifeline, - WeakrefLifeline) - if lifeline is not None: - lifeline.ref_is_dead(w_self.index) class W_Weakref(W_WeakrefBase): - def __init__(w_self, space, lifeline, index, w_obj, w_callable): - W_WeakrefBase.__init__(w_self, space, lifeline, index, w_obj, w_callable) + def __init__(w_self, space, w_obj, w_callable): + W_WeakrefBase.__init__(w_self, space, w_obj, w_callable) w_self.w_hash = None def descr_hash(self): @@ -127,13 +109,15 @@ if lifeline is None: lifeline = WeakrefLifeline() w_obj.setweakref(space, lifeline) - return lifeline.get_weakref(space, w_subtype, w_obj, w_callable) + return lifeline.get_or_make_weakref(space, w_subtype, w_obj, w_callable) def descr__eq__(space, ref1, ref2): - if (cast_weakgcaddress_to_object(ref1.address, W_Root) is None or - cast_weakgcaddress_to_object(ref2.address, W_Root) is None): + w_obj1 = ref1.dereference() + w_obj2 = ref2.dereference() + if (space.is_w(w_obj1, space.w_None) or + space.is_w(w_obj2, space.w_None)): return space.is_(ref1, ref2) - return space.eq(ref1.dereference(), ref2.dereference()) + return space.eq(w_obj1, w_obj2) def descr__ne__(space, ref1, ref2): return space.not_(space.eq(ref1, ref2)) @@ -159,8 +143,8 @@ return space.wrap(0) else: result = 0 - for i in range(len(lifeline.addr_refs)): - if cast_weakgcaddress_to_object(lifeline.addr_refs[i], W_Root) is not None: + for i in range(len(lifeline.refs_weak)): + if lifeline.refs_weak[i]() is not None: result += 1 return space.wrap(result) @@ -171,10 +155,10 @@ return space.newlist([]) else: result = [] - for i in range(len(lifeline.addr_refs)): - addr = lifeline.addr_refs[i] - if cast_weakgcaddress_to_object(addr, W_Root) is not None: - result.append(cast_weakgcaddress_to_object(addr, W_Root)) + for i in range(len(lifeline.refs_weak)): + w_ref = lifeline.refs_weak[i]() + if w_ref is not None: + result.append(w_ref) return space.newlist(result) #_________________________________________________________________ @@ -198,7 +182,7 @@ if lifeline is None: lifeline = WeakrefLifeline() w_obj.setweakref(space, lifeline) - return lifeline.get_proxy(space, w_obj, w_callable) + return lifeline.get_or_make_proxy(space, w_obj, w_callable) def descr__new__proxy(space, w_subtype, w_obj, w_callable=None): raise OperationError( @@ -255,21 +239,3 @@ unwrap_spec=['self', ObjSpace, Arguments]), **callable_proxy_typedef_dict) W_CallableProxy.typedef.acceptable_as_base_class = False - -def basic_weakref(space, w_obj): - """this is a bit like the app-level weakref.ref(), but with no - fancy options like supporting subclasses of _weakref.ref and - callbacks.""" - lifeline = w_obj.getweakref() - if lifeline is None: - lifeline = WeakrefLifeline() - w_obj.setweakref(space, lifeline) - if lifeline.cached_weakref_index >= 0: - cached_weakref_address = lifeline.addr_refs[lifeline.cached_weakref_index] - return cast_weakgcaddress_to_object(cached_weakref_address, W_Weakref) - index = len(lifeline.addr_refs) - w_ref = W_Weakref(space, lifeline, index, w_obj, space.w_None) - lifeline.addr_refs.append(cast_object_to_weakgcaddress(w_ref)) - lifeline.cached_weakref_index = index - return w_ref - From cfbolz at codespeak.net Tue Sep 11 17:27:37 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Sep 2007 17:27:37 +0200 (CEST) Subject: [pypy-svn] r46475 - in pypy/dist/pypy: module/_weakref/test objspace/std Message-ID: <20070911152737.79C4A8101@code0.codespeak.net> Author: cfbolz Date: Tue Sep 11 17:27:36 2007 New Revision: 46475 Modified: pypy/dist/pypy/module/_weakref/test/test_weakref.py pypy/dist/pypy/objspace/std/setobject.py Log: enable taking weakrefs of sets Modified: pypy/dist/pypy/module/_weakref/test/test_weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/test/test_weakref.py (original) +++ pypy/dist/pypy/module/_weakref/test/test_weakref.py Tue Sep 11 17:27:36 2007 @@ -209,7 +209,6 @@ assert w_bound() is None def test_set_weakrefable(self): - skip("missing: weakrefs to interp-level sets") import _weakref, gc s = set([1, 2, 3, 4]) w = _weakref.ref(s) Modified: pypy/dist/pypy/objspace/std/setobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/setobject.py (original) +++ pypy/dist/pypy/objspace/std/setobject.py Tue Sep 11 17:27:36 2007 @@ -29,6 +29,12 @@ obj = space.call_function(space.type(w_self),itemiterator) return obj + _lifeline_ = None + def getweakref(self): + return self._lifeline_ + def setweakref(self, space, weakreflifeline): + self._lifeline_ = weakreflifeline + class W_SetObject(W_BaseSetObject): from pypy.objspace.std.settype import set_typedef as typedef From antocuni at codespeak.net Tue Sep 11 18:43:56 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 11 Sep 2007 18:43:56 +0200 (CEST) Subject: [pypy-svn] r46476 - in pypy/dist/pypy/translator: cli/test jvm/test oosupport/test_template Message-ID: <20070911164356.E30F380E3@code0.codespeak.net> Author: antocuni Date: Tue Sep 11 18:43:55 2007 New Revision: 46476 Added: pypy/dist/pypy/translator/oosupport/test_template/builtin.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py pypy/dist/pypy/translator/jvm/test/runtest.py pypy/dist/pypy/translator/jvm/test/test_builtin.py Log: move test_builtin to oosupport Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/cli/test/test_builtin.py Tue Sep 11 18:43:55 2007 @@ -1,11 +1,8 @@ import platform -import os, stat, errno import py -from py.builtin import sorted -from pypy.tool import udir from pypy.translator.cli.test.runtest import CliTest -from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin -from pypy.rpython.module.test.test_ll_time import BaseTestTime +from pypy.translator.oosupport.test_template.builtin import BaseTestBuiltin, BaseTestTime + def skip_os(self): py.test.skip("CLI doesn't support the os module, yet") @@ -14,138 +11,15 @@ if platform.system() == 'Windows': py.test.skip("Doesn't work on Windows, yet") -class TestCliBuiltin(CliTest, BaseTestRbuiltin): +class TestCliBuiltin(CliTest, BaseTestBuiltin): test_os_dup = skip_os test_os_path_exists = skip_os test_os_isdir = skip_os def test_builtin_math_frexp(self): self._skip_powerpc("Mono math floating point problem") - BaseTestRbuiltin.test_builtin_math_frexp(self) - - def test_os_flags(self): - from pypy.translator.cli.support import NT_OS - def fn(): - return os.O_CREAT - assert self.interpret(fn, []) == NT_OS['O_CREAT'] - - def test_os_read(self): - BaseTestRbuiltin.test_os_read(self) - - def test_os_read_binary_crlf(self): - tmpfile = str(udir.udir.join("os_read_test")) - def fn(flag): - if flag: - fd = os.open(tmpfile, os.O_RDONLY|os.O_BINARY, 0666) - else: - fd = os.open(tmpfile, os.O_RDONLY, 0666) - res = os.read(fd, 4096) - os.close(fd) - return res - f = file(tmpfile, 'w') - f.write('Hello\nWorld') - f.close() - res = self.ll_to_string(self.interpret(fn, [True])) - assert res == file(tmpfile, 'rb').read() - res = self.ll_to_string(self.interpret(fn, [False])) - assert res == file(tmpfile, 'r').read() - - # the following tests can't be executed with gencli because they - # returns file descriptors, and cli code is executed in another - # process. Instead of those, there is a new test that opens and - # write to a file all in the same process. - def test_os_write(self): - pass - def test_os_write_single_char(self): - pass - def test_os_open(self): - pass - - def test_os_open_write(self): - tmpdir = str(udir.udir.join("os_write_test")) - def fn(): - fd = os.open(tmpdir, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0777) - os.write(fd, "hello world") - os.close(fd) - self.interpret(fn, []) - assert file(tmpdir).read() == 'hello world' - - def test_os_write_magic(self): - MAGIC = 62061 | (ord('\r')<<16) | (ord('\n')<<24) - tmpfile = str(udir.udir.join("os_write_test")) - def long2str(x): - a = x & 0xff - x >>= 8 - b = x & 0xff - x >>= 8 - c = x & 0xff - x >>= 8 - d = x & 0xff - return chr(a) + chr(b) + chr(c) + chr(d) - def fn(magic): - fd = os.open(tmpfile, os.O_BINARY|os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0777) - os.write(fd, long2str(magic)) - os.close(fd) - self.interpret(fn, [MAGIC]) - contents = file(tmpfile, 'rb').read() - assert contents == long2str(MAGIC) - - def test_os_stat(self): - def fn(flag): - if flag: - return os.stat('.')[0] - else: - return os.stat('.').st_mode - mode = self.interpret(fn, [0]) - assert stat.S_ISDIR(mode) - mode = self.interpret(fn, [1]) - assert stat.S_ISDIR(mode) - - def test_os_stat_oserror(self): - def fn(): - return os.stat('/directory/unlikely/to/exists')[0] - self.interpret_raises(OSError, fn, []) - - def test_os_strerror(self): - def fn(): - return os.strerror(errno.ENOTDIR) - res = self.ll_to_string(self.interpret(fn, [])) - # XXX assert something about res - - def test_environ(self): - def fn(): - os.environ['PYPY_TEST_ENVIRON'] = '42' - return os.environ['PYPY_TEST_ENVIRON'] - assert self.interpret(fn, []) == '42' - - def test_environ_items(self): - def fn(): - env = os.environ.items() - env2 = [] - for key in os.environ.keys(): - env2.append((key, os.environ[key])) - assert env == env2 - self.interpret(fn, []) - - def test_os_listdir(self): - def fn(): - return os.listdir('.') - res = self.ll_to_list(self.interpret(fn, [])) - res = [self.ll_to_string(s) for s in res] - res.sort() - assert res == sorted(os.listdir('.')) - - # XXX: remember to test ll_os_readlink and ll_os_pipe as soon as - # they are implemented + BaseTestBuiltin.test_builtin_math_frexp(self) class TestCliTime(CliTest, BaseTestTime): - - def test_time_clock(self): - import time - def f(): - return time.clock(), time.clock(), time.clock() - res = self.interpret(f, []) - t1, t2, t3 = self.ll_to_tuple(res) - assert 0 <= t1 <= t2 <= t3 - + pass Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Tue Sep 11 18:43:55 2007 @@ -140,6 +140,9 @@ def ll_to_list(self, l): return l + def ll_to_tuple(self, t): + return t + def class_name(self, value): return value.class_name.split(".")[-1] Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Tue Sep 11 18:43:55 2007 @@ -1,40 +1,21 @@ -import os, stat + import py -from pypy.tool import udir +from pypy.translator.oosupport.test_template.builtin import BaseTestBuiltin, BaseTestTime from pypy.translator.jvm.test.runtest import JvmTest -from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin -class TestJavaBuiltin(JvmTest, BaseTestRbuiltin): - def test_os(self): - py.test.skip("Jvm os support uncertain") - - def test_os_open(self): - py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - - def test_os_write(self): +class TestJavaBuiltin(JvmTest, BaseTestBuiltin): + + def test_os_flags(self): + py.test.skip('fixme!') + + def test_os_open_write(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - - def test_os_write_single_char(self): + + def test_os_write_magic(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - + def test_os_read(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - - def test_os_stat(self): - def fn(flag): - if flag: - return os.stat('.')[0] - else: - return os.stat('.').st_mode - mode = self.interpret(fn, [0]) - assert stat.S_ISDIR(mode) - mode = self.interpret(fn, [1]) - assert stat.S_ISDIR(mode) - - def test_os_stat_oserror(self): - def fn(): - return os.stat('/directory/unlikely/to/exists')[0] - self.interpret_raises(OSError, fn, []) def test_builtin_math_frexp(self): py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") @@ -44,3 +25,23 @@ def test_os_dup(self): py.test.skip("not implemented") + + def test_environ_items(self): + py.test.skip('fixme!') + + def test_environ(self): + py.test.skip('fixme!') + + def test_os_listdir(self): + py.test.skip('fixme!') + + def test_os_read_binary_crlf(self): + py.test.skip('fixme!') + + + +class TestJvmTime(JvmTest, BaseTestTime): + + def test_time_sleep(self): + py.test.skip('fixme!') + Added: pypy/dist/pypy/translator/oosupport/test_template/builtin.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/test_template/builtin.py Tue Sep 11 18:43:55 2007 @@ -0,0 +1,132 @@ +import os +import errno +import stat +from py.builtin import sorted +from pypy.tool import udir +from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin +from pypy.rpython.module.test.test_ll_time import BaseTestTime as llBaseTestTime + +class BaseTestBuiltin(BaseTestRbuiltin): + + def test_os_flags(self): + from pypy.translator.cli.support import NT_OS + def fn(): + return os.O_CREAT + assert self.interpret(fn, []) == NT_OS['O_CREAT'] + + def test_os_read_binary_crlf(self): + tmpfile = str(udir.udir.join("os_read_test")) + def fn(flag): + if flag: + fd = os.open(tmpfile, os.O_RDONLY|os.O_BINARY, 0666) + else: + fd = os.open(tmpfile, os.O_RDONLY, 0666) + res = os.read(fd, 4096) + os.close(fd) + return res + f = file(tmpfile, 'w') + f.write('Hello\nWorld') + f.close() + res = self.ll_to_string(self.interpret(fn, [True])) + assert res == file(tmpfile, 'rb').read() + res = self.ll_to_string(self.interpret(fn, [False])) + assert res == file(tmpfile, 'r').read() + + # the following tests can't be executed with gencli because they + # returns file descriptors, and cli code is executed in another + # process. Instead of those, there is a new test that opens and + # write to a file all in the same process. + def test_os_write(self): + pass + def test_os_write_single_char(self): + pass + def test_os_open(self): + pass + + def test_os_open_write(self): + tmpdir = str(udir.udir.join("os_write_test")) + def fn(): + fd = os.open(tmpdir, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0777) + os.write(fd, "hello world") + os.close(fd) + self.interpret(fn, []) + assert file(tmpdir).read() == 'hello world' + + def test_os_write_magic(self): + MAGIC = 62061 | (ord('\r')<<16) | (ord('\n')<<24) + tmpfile = str(udir.udir.join("os_write_test")) + def long2str(x): + a = x & 0xff + x >>= 8 + b = x & 0xff + x >>= 8 + c = x & 0xff + x >>= 8 + d = x & 0xff + return chr(a) + chr(b) + chr(c) + chr(d) + def fn(magic): + fd = os.open(tmpfile, os.O_BINARY|os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0777) + os.write(fd, long2str(magic)) + os.close(fd) + self.interpret(fn, [MAGIC]) + contents = file(tmpfile, 'rb').read() + assert contents == long2str(MAGIC) + + def test_os_stat(self): + def fn(flag): + if flag: + return os.stat('.')[0] + else: + return os.stat('.').st_mode + mode = self.interpret(fn, [0]) + assert stat.S_ISDIR(mode) + mode = self.interpret(fn, [1]) + assert stat.S_ISDIR(mode) + + def test_os_stat_oserror(self): + def fn(): + return os.stat('/directory/unlikely/to/exists')[0] + self.interpret_raises(OSError, fn, []) + + def test_os_strerror(self): + def fn(): + return os.strerror(errno.ENOTDIR) + res = self.ll_to_string(self.interpret(fn, [])) + # XXX assert something about res + + def test_environ(self): + def fn(): + os.environ['PYPY_TEST_ENVIRON'] = '42' + return os.environ['PYPY_TEST_ENVIRON'] + assert self.interpret(fn, []) == '42' + + def test_environ_items(self): + def fn(): + env = os.environ.items() + env2 = [] + for key in os.environ.keys(): + env2.append((key, os.environ[key])) + assert env == env2 + self.interpret(fn, []) + + def test_os_listdir(self): + def fn(): + return os.listdir('.') + res = self.ll_to_list(self.interpret(fn, [])) + res = [self.ll_to_string(s) for s in res] + res.sort() + assert res == sorted(os.listdir('.')) + + # XXX: remember to test ll_os_readlink and ll_os_pipe as soon as + # they are implemented + + +class BaseTestTime(llBaseTestTime): + + def test_time_clock(self): + import time + def f(): + return time.clock(), time.clock(), time.clock() + res = self.interpret(f, []) + t1, t2, t3 = self.ll_to_tuple(res) + assert 0 <= t1 <= t2 <= t3 From cfbolz at codespeak.net Tue Sep 11 19:06:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Sep 2007 19:06:03 +0200 (CEST) Subject: [pypy-svn] r46477 - pypy/branch/cleanup-weakref Message-ID: <20070911170603.AB9F080EC@code0.codespeak.net> Author: cfbolz Date: Tue Sep 11 19:06:02 2007 New Revision: 46477 Added: pypy/branch/cleanup-weakref/ - copied from r46476, pypy/dist/ Log: short-lived branch for cleaning up weakrefs From cfbolz at codespeak.net Tue Sep 11 19:09:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Sep 2007 19:09:46 +0200 (CEST) Subject: [pypy-svn] r46478 - in pypy/branch/cleanup-weakref/pypy: annotation rlib rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test translator translator/c translator/c/src translator/c/test translator/cli translator/jvm translator/llvm/test translator/oosupport/test_template translator/stackless Message-ID: <20070911170946.928BC8104@code0.codespeak.net> Author: cfbolz Date: Tue Sep 11 19:09:45 2007 New Revision: 46478 Modified: pypy/branch/cleanup-weakref/pypy/annotation/binaryop.py pypy/branch/cleanup-weakref/pypy/annotation/bookkeeper.py pypy/branch/cleanup-weakref/pypy/annotation/builtin.py pypy/branch/cleanup-weakref/pypy/annotation/model.py pypy/branch/cleanup-weakref/pypy/rlib/objectmodel.py pypy/branch/cleanup-weakref/pypy/rlib/test/test_objectmodel.py pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/opimpl.py pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/test/test_llmemory.py pypy/branch/cleanup-weakref/pypy/rpython/raddress.py pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py pypy/branch/cleanup-weakref/pypy/rpython/rmodel.py pypy/branch/cleanup-weakref/pypy/translator/c/funcgen.py pypy/branch/cleanup-weakref/pypy/translator/c/node.py pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py pypy/branch/cleanup-weakref/pypy/translator/c/src/address.h pypy/branch/cleanup-weakref/pypy/translator/c/test/test_lladdresses.py pypy/branch/cleanup-weakref/pypy/translator/c/test/test_newgc.py pypy/branch/cleanup-weakref/pypy/translator/cli/metavm.py pypy/branch/cleanup-weakref/pypy/translator/cli/opcodes.py pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py pypy/branch/cleanup-weakref/pypy/translator/jvm/metavm.py pypy/branch/cleanup-weakref/pypy/translator/jvm/opcodes.py pypy/branch/cleanup-weakref/pypy/translator/llvm/test/test_lladdresses.py pypy/branch/cleanup-weakref/pypy/translator/oosupport/test_template/objectmodel.py pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py Log: yay! kill the ugly and unsafe casts from and to weak addresses Modified: pypy/branch/cleanup-weakref/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/annotation/binaryop.py (original) +++ pypy/branch/cleanup-weakref/pypy/annotation/binaryop.py Tue Sep 11 19:09:45 2007 @@ -14,7 +14,6 @@ from pypy.annotation.model import SomeExternalObject from pypy.annotation.model import SomeWeakRef, SomeDeadWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeWeakGcAddress from pypy.annotation.model import SomeCTypesObject from pypy.annotation.model import unionof, UnionError, set, missing_operation, TLS from pypy.annotation.model import read_can_only_throw @@ -949,10 +948,6 @@ raise UnionError, "union of address and anything else makes no sense" -class __extend__(pairtype(SomeWeakGcAddress, SomeWeakGcAddress)): - def union((s_addr1, s_addr2)): - return SomeWeakGcAddress() - class __extend__(pairtype(SomeCTypesObject, SomeInteger)): def setitem((s_cto, s_index), s_value): Modified: pypy/branch/cleanup-weakref/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/cleanup-weakref/pypy/annotation/bookkeeper.py Tue Sep 11 19:09:45 2007 @@ -11,8 +11,7 @@ SomeInteger, SomeExternalObject, SomeOOInstance, TLS, SomeAddress, \ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ - SomeList, SomeObject, SomeWeakGcAddress, HarmlesslyBlocked, \ - SomeWeakRef, SomeDeadWeakRef + SomeList, SomeObject, HarmlesslyBlocked, SomeWeakRef, SomeDeadWeakRef from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -409,8 +408,6 @@ result = SomePtr(lltype.typeOf(x)) elif isinstance(x, llmemory.fakeaddress): result = SomeAddress(is_null=not x) - elif isinstance(x, llmemory.fakeweakaddress): - result = SomeWeakGcAddress() elif isinstance(x, ootype._static_meth): result = SomeOOStaticMeth(ootype.typeOf(x)) elif isinstance(x, ootype._class): Modified: pypy/branch/cleanup-weakref/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/annotation/builtin.py (original) +++ pypy/branch/cleanup-weakref/pypy/annotation/builtin.py Tue Sep 11 19:09:45 2007 @@ -6,7 +6,7 @@ from pypy.annotation.model import SomeInteger, SomeObject, SomeChar, SomeBool from pypy.annotation.model import SomeString, SomeTuple, SomeSlice, s_Bool from pypy.annotation.model import SomeUnicodeCodePoint, SomeAddress -from pypy.annotation.model import SomeFloat, SomeWeakGcAddress, unionof +from pypy.annotation.model import SomeFloat, unionof from pypy.annotation.model import SomePBC, SomeInstance, SomeDict from pypy.annotation.model import SomeExternalObject from pypy.annotation.model import SomeWeakRef, SomeLLWeakRef @@ -331,13 +331,6 @@ assert s_type.is_constant() return SomePtr(s_type.const) -def llmemory_cast_ptr_to_weakadr(s): - return SomeWeakGcAddress() - -def llmemory_cast_weakadr_to_ptr(s, s_type): - assert s_type.is_constant() - return SomePtr(s_type.const) - def llmemory_cast_adr_to_int(s): return SomeInteger() # xxx @@ -389,8 +382,6 @@ BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_ptr] = llmemory_cast_adr_to_ptr BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_int] = llmemory_cast_adr_to_int BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_int_to_adr] = llmemory_cast_int_to_adr -BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_ptr_to_weakadr] = llmemory_cast_ptr_to_weakadr -BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_weakadr_to_ptr] = llmemory_cast_weakadr_to_ptr BUILTIN_ANALYZERS[pypy.rlib.rstack.yield_current_frame_to_caller] = ( rstack_yield_current_frame_to_caller) Modified: pypy/branch/cleanup-weakref/pypy/annotation/model.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/annotation/model.py (original) +++ pypy/branch/cleanup-weakref/pypy/annotation/model.py Tue Sep 11 19:09:45 2007 @@ -529,9 +529,6 @@ def can_be_none(self): return False -class SomeWeakGcAddress(SomeObject): - immutable = True - # The following class is used to annotate the intermediate value that # appears in expressions of the form: # addr.signed[offset] and addr.signed[offset] = value @@ -596,7 +593,6 @@ (SomeChar(), lltype.Char), (SomeUnicodeCodePoint(), lltype.UniChar), (SomeAddress(), llmemory.Address), - (SomeWeakGcAddress(), llmemory.WeakGcAddress), (SomeLLWeakRef(), llmemory.WeakRef), ] Modified: pypy/branch/cleanup-weakref/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rlib/objectmodel.py (original) +++ pypy/branch/cleanup-weakref/pypy/rlib/objectmodel.py Tue Sep 11 19:09:45 2007 @@ -123,70 +123,8 @@ obj.__dict__ = {} obj.__class__ = FREED_OBJECT -def cast_object_to_weakgcaddress(obj): - from pypy.rpython.lltypesystem.llmemory import fakeweakaddress - return fakeweakaddress(obj) - -def cast_weakgcaddress_to_object(address, expected_result): - if address.ref is None: # NULL address - return None - obj = address.get() - assert obj is not None - assert isinstance(obj, expected_result) - return obj - from pypy.rpython.extregistry import ExtRegistryEntry -class Entry(ExtRegistryEntry): - _about_ = cast_object_to_weakgcaddress - - def compute_result_annotation(self, s_obj): - from pypy.annotation import model as annmodel - return annmodel.SomeWeakGcAddress() - - def specialize_call(self, hop): - vlist = hop.inputargs(hop.args_r[0]) - return hop.genop('cast_ptr_to_weakadr', vlist, - resulttype=hop.r_result.lowleveltype) - -class Entry(ExtRegistryEntry): - _about_ = cast_weakgcaddress_to_object - - def compute_result_annotation(self, s_int, s_clspbc): - from pypy.annotation import model as annmodel - assert len(s_clspbc.descriptions) == 1 - desc = s_clspbc.descriptions.keys()[0] - cdef = desc.getuniqueclassdef() - return annmodel.SomeInstance(cdef, can_be_None=True) - - def specialize_call(self, hop): - from pypy.rpython import raddress - assert isinstance(hop.args_r[0], raddress.WeakGcAddressRepr) - vlist = [hop.inputarg(hop.args_r[0], arg=0)] - return hop.genop('cast_weakadr_to_ptr', vlist, - resulttype = hop.r_result.lowleveltype) - - -def cast_weakgcaddress_to_int(address): - if address.ref is None: # NULL address - return 0 - return address.cast_to_int() - - -class Entry(ExtRegistryEntry): - _about_ = cast_weakgcaddress_to_int - - def compute_result_annotation(self, s_int): - return annmodel.SomeInteger() - - - def specialize_call(self, hop): - from pypy.rpython import raddress - assert isinstance(hop.args_r[0], raddress.WeakGcAddressRepr) - vlist = [hop.inputarg(hop.args_r[0], arg=0)] - return hop.genop('cast_weakadr_to_int', vlist, - resulttype = hop.r_result.lowleveltype) - # ____________________________________________________________ def debug_assert(x, msg): Modified: pypy/branch/cleanup-weakref/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/cleanup-weakref/pypy/rlib/test/test_objectmodel.py Tue Sep 11 19:09:45 2007 @@ -271,28 +271,6 @@ class TestLLtype(BaseTestObjectModel, LLRtypeMixin): - def test_cast_to_and_from_weakaddress(self): - class A(object): - pass - class B(object): - pass - def f(): - a = A() - addr = cast_object_to_weakgcaddress(a) - return a is cast_weakgcaddress_to_object(addr, A) - assert f() - res = self.interpret(f, []) - assert res - a = A() - addr = cast_object_to_weakgcaddress(A) - py.test.raises(AssertionError, "cast_weakgcaddress_to_object(addr, B)") - assert isinstance(cast_weakgcaddress_to_int(addr), int) - def g(): - a = A() - addr = cast_object_to_weakgcaddress(a) - return cast_weakgcaddress_to_int(addr) - assert isinstance(self.interpret(f, []), int) - def test_rtype_keepalive(self): from pypy.rlib import objectmodel def f(): Modified: pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py Tue Sep 11 19:09:45 2007 @@ -494,11 +494,10 @@ # target malloc'ed structure alive. When the target dies # all WeakRefs to it are cleared. # -# WeakGcAddress - like WeakRef but not automatically cleared. There is -# no direct way to know if the target is still alive. -# We should more or less deprecate them in favor of WeakRef. -class fakeweakaddress(object): +# ____________________________________________________________ + +class fakeweakref(object): # XXX convoluted code to support both lltype._ptr and simulatorptr def __init__(self, ob): if ob is not None: @@ -531,32 +530,11 @@ else: s = str(self.obref) return '<%s %s>' % (self.__class__.__name__, s) - def cast_to_int(self): - # this is not always the behaviour that is really happening - # but make sure that nobody depends on it - return self.id ^ ~3 - -WeakGcAddress = lltype.Primitive("WeakGcAddress", - fakeweakaddress(None)) - -def cast_ptr_to_weakadr(obj): - assert isinstance(lltype.typeOf(obj), lltype.Ptr) - return fakeweakaddress(obj) - -def cast_weakadr_to_ptr(adr, EXPECTED_TYPE): - result = adr.get() - if result is None: - return lltype.nullptr(EXPECTED_TYPE.TO) - else: - return lltype.cast_pointer(EXPECTED_TYPE, result) - -fakeweakaddress._TYPE = WeakGcAddress -WEAKNULL = fakeweakaddress(None) - -# ____________________________________________________________ + #def cast_to_int(self): + # # this is not always the behaviour that is really happening + # # but make sure that nobody depends on it + # return self.id ^ ~3 -class fakeweakref(fakeweakaddress): - pass # inheriting only to copy all methods WeakRef = lltype.Primitive("WeakRef", fakeweakref(None)) Modified: pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py Tue Sep 11 19:09:45 2007 @@ -362,9 +362,6 @@ 'adr_call': LLOp(canraise=(Exception,)), 'cast_ptr_to_adr': LLOp(canfold=True), 'cast_adr_to_ptr': LLOp(canfold=True), - 'cast_ptr_to_weakadr': LLOp(canfold=True), - 'cast_weakadr_to_ptr': LLOp(canfold=True), - 'cast_weakadr_to_int': LLOp(canfold=True), 'cast_adr_to_int': LLOp(canfold=True), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp Modified: pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/opimpl.py Tue Sep 11 19:09:45 2007 @@ -276,19 +276,6 @@ return lltype.cast_pointer(RESTYPE, obj) op_cast_pointer.need_result_type = True -def op_cast_ptr_to_weakadr(ptr): - checkptr(ptr) - return llmemory.cast_ptr_to_weakadr(ptr) - -def op_cast_weakadr_to_ptr(TYPE, wadr): - assert lltype.typeOf(wadr) == llmemory.WeakGcAddress - return llmemory.cast_weakadr_to_ptr(wadr, TYPE) -op_cast_weakadr_to_ptr.need_result_type = True - -def op_cast_weakadr_to_int(wadr): - assert lltype.typeOf(wadr) == llmemory.WeakGcAddress - return wadr.cast_to_int() - def op_cast_ptr_to_adr(ptr): checkptr(ptr) return llmemory.cast_ptr_to_adr(ptr) Modified: pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/test/test_llmemory.py Tue Sep 11 19:09:45 2007 @@ -157,35 +157,6 @@ res = interpret(f, []) assert res -def test_weak_casts(): - from pypy.rpython.memory.test.test_llinterpsim import interpret - S = lltype.GcStruct("S", ("x", lltype.Signed)) - Sptr = lltype.Ptr(S) - def f(): - s1 = lltype.malloc(S) - adr = cast_ptr_to_weakadr(s1) - s2 = cast_weakadr_to_ptr(adr, Sptr) - return s1 == s2 - res = interpret(f, []) - assert res - -def test_weak_casts_lifetime(): - T = lltype.GcStruct("T", ("x", lltype.Signed)) - S = lltype.GcStruct("S", ("t", T)) - Sptr = lltype.Ptr(S) - def g(): - s1 = lltype.malloc(S) - adr = cast_ptr_to_weakadr(s1) - s2 = cast_weakadr_to_ptr(adr, Sptr) - assert s1 == s2 - return adr, lltype.cast_pointer(lltype.Ptr(T), s2) # s1, s2 go away - def f(): - adr, t = g() - t2 = cast_weakadr_to_ptr(adr, lltype.Ptr(T)) - return t2 == t - res = interpret(f, []) - assert res - def test_fakeaccessor(): S = lltype.GcStruct("S", ("x", lltype.Signed), ("y", lltype.Signed)) s = lltype.malloc(S) @@ -609,31 +580,3 @@ assert weakref_deref(lltype.Ptr(S), w) == lltype.nullptr(S) assert weakref_deref(lltype.Ptr(S1), w) == lltype.nullptr(S1) -class TestWeakAddressLLinterp(object): - def test_null(self): - from pypy.rlib.objectmodel import cast_weakgcaddress_to_object - from pypy.rlib.objectmodel import cast_object_to_weakgcaddress - class A: - pass - def f(): - return cast_weakgcaddress_to_object(WEAKNULL, A) is None - assert interpret(f, []) - - def test_attribute(object): - from pypy.rlib.objectmodel import cast_weakgcaddress_to_object - from pypy.rlib.objectmodel import cast_object_to_weakgcaddress - class A: - pass - class B: - pass - def f(x): - a = A() - b = B() - if x: - a.addr = WEAKNULL - else: - a.addr = cast_object_to_weakgcaddress(b) - return cast_weakgcaddress_to_object(a.addr, B) is b - assert not interpret(f, [1]) - assert interpret(f, [0]) - Modified: pypy/branch/cleanup-weakref/pypy/rpython/raddress.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/raddress.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/raddress.py Tue Sep 11 19:09:45 2007 @@ -3,7 +3,7 @@ from pypy.annotation import model as annmodel from pypy.rpython.memory.lladdress import _address from pypy.rpython.lltypesystem.llmemory import NULL, Address, \ - cast_adr_to_int, WeakGcAddress + cast_adr_to_int from pypy.rpython.rmodel import Repr, IntegerRepr from pypy.rpython.rptr import PtrRepr from pypy.rpython.lltypesystem import lltype @@ -16,13 +16,6 @@ def rtyper_makekey(self): return self.__class__, -class __extend__(annmodel.SomeWeakGcAddress): - def rtyper_makerepr(self, rtyper): - return WeakGcAddressRepr(rtyper) - - def rtyper_makekey(self): - return self.__class__, - class __extend__(annmodel.SomeTypedAddressAccess): def rtyper_makerepr(self, rtyper): return TypedAddressAccessRepr(self.type) @@ -143,27 +136,4 @@ def convert_from_to((r_ptr, r_addr), v, llops): return llops.genop('cast_ptr_to_adr', [v], resulttype=Address) -class WeakGcAddressRepr(Repr): - lowleveltype = WeakGcAddress - - def __init__(self, rtyper): - self.rtyper = rtyper - def convert_const(self, value): - from pypy.rpython.lltypesystem import llmemory - from pypy.rlib.objectmodel import cast_object_to_weakgcaddress - from pypy.objspace.flow.model import Constant - assert isinstance(value, llmemory.fakeweakaddress) - if value.ref is None: - return value - ob = value.ref() - assert ob is not None - bk = self.rtyper.annotator.bookkeeper - # obscure! if the annotator hasn't seen this object before, - # we don't want to look at it now (confusion tends to result). - if bk.have_seen(ob): - repr = self.rtyper.bindingrepr(Constant(ob)) - newob = repr.convert_const(ob) - return cast_object_to_weakgcaddress(newob) - else: - return llmemory.fakeweakaddress(None) Modified: pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py Tue Sep 11 19:09:45 2007 @@ -661,24 +661,9 @@ return hop.genop('cast_int_to_adr', [v_input], resulttype = llmemory.Address) -def rtype_cast_ptr_to_weakadr(hop): - vlist = hop.inputargs(hop.args_r[0]) - assert isinstance(vlist[0].concretetype, lltype.Ptr) - hop.exception_cannot_occur() - return hop.genop('cast_ptr_to_weakadr', vlist, - resulttype = llmemory.WeakGcAddress) - -def rtype_cast_weakadr_to_ptr(hop): - assert isinstance(hop.args_r[0], raddress.WeakGcAddressRepr) - adr, TYPE = hop.inputargs(hop.args_r[0], lltype.Void) - hop.exception_cannot_occur() - return hop.genop('cast_weakadr_to_ptr', [adr], - resulttype = TYPE.value) BUILTIN_TYPER[llmemory.cast_ptr_to_adr] = rtype_cast_ptr_to_adr BUILTIN_TYPER[llmemory.cast_adr_to_ptr] = rtype_cast_adr_to_ptr BUILTIN_TYPER[llmemory.cast_adr_to_int] = rtype_cast_adr_to_int BUILTIN_TYPER[llmemory.cast_int_to_adr] = rtype_cast_int_to_adr -BUILTIN_TYPER[llmemory.cast_ptr_to_weakadr] = rtype_cast_ptr_to_weakadr -BUILTIN_TYPER[llmemory.cast_weakadr_to_ptr] = rtype_cast_weakadr_to_ptr Modified: pypy/branch/cleanup-weakref/pypy/rpython/rmodel.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/rmodel.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/rmodel.py Tue Sep 11 19:09:45 2007 @@ -220,11 +220,8 @@ raise TyperError('id() of an instance of the non-pointer %r' % ( self,)) vobj, = hop.inputargs(self) - # XXX why did this go through weakadr?? - #v_waddr = hop.genop('cast_ptr_to_weakadr', [vobj], - # resulttype=llmemory.WeakGcAddress) - #return hop.genop('cast_weakadr_to_int', [v_waddr], resulttype=Signed) - + # XXX I am still not sure whether this is the right thing to do, + # since the int might keep things alive with boehm return hop.genop('cast_ptr_to_int', [vobj], resulttype=Signed) def rtype_hash(self, hop): Modified: pypy/branch/cleanup-weakref/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/funcgen.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/funcgen.py Tue Sep 11 19:09:45 2007 @@ -11,7 +11,7 @@ from pypy.rpython.lltypesystem.lltype import pyobjectptr, ContainerType from pypy.rpython.lltypesystem.lltype import Struct, Array, FixedSizeArray from pypy.rpython.lltypesystem.lltype import ForwardReference, FuncType -from pypy.rpython.lltypesystem.llmemory import Address, WeakGcAddress +from pypy.rpython.lltypesystem.llmemory import Address from pypy.translator.backendopt.ssa import SSI_to_SSA PyObjPtr = Ptr(PyObject) @@ -568,18 +568,6 @@ OP_CAST_ADR_TO_PTR = OP_CAST_POINTER OP_CAST_OPAQUE_PTR = OP_CAST_POINTER - def OP_CAST_PTR_TO_WEAKADR(self, op): - return '%s = HIDE_POINTER(%s);' % (self.expr(op.result), - self.expr(op.args[0])) - - def OP_CAST_WEAKADR_TO_PTR(self, op): - TYPE = self.lltypemap(op.result) - assert TYPE != PyObjPtr - typename = self.db.gettype(TYPE) - return '%s = (%s)REVEAL_POINTER(%s);' % (self.expr(op.result), - cdecl(typename, ''), - self.expr(op.args[0])) - def OP_CAST_INT_TO_PTR(self, op): TYPE = self.lltypemap(op.result) typename = self.db.gettype(TYPE) @@ -652,7 +640,7 @@ format.append('%d') elif T == Float: format.append('%f') - elif isinstance(T, Ptr) or T in (Address, WeakGcAddress): + elif isinstance(T, Ptr) or T == Address: format.append('%p') elif T == Char: if isinstance(arg, Constant): Modified: pypy/branch/cleanup-weakref/pypy/translator/c/node.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/node.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/node.py Tue Sep 11 19:09:45 2007 @@ -4,7 +4,6 @@ GcStruct, GcArray, RttiStruct, PyStruct, ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray, _pyobjheader -from pypy.rpython.lltypesystem.llmemory import WeakGcAddress from pypy.translator.c.funcgen import FunctionCodeGenerator from pypy.translator.c.external import CExternalFunctionCodeGenerator from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring @@ -593,9 +592,6 @@ db.late_initializations.append(('%s' % access_expr, db.get(value))) expr = '0.0 /* patched later by %sinfinity */' % ( '-+'[value > 0]) - elif typeOf(value) == WeakGcAddress and value.ref is not None: - db.late_initializations.append(('%s' % access_expr, db.get(value))) - expr = 'HIDE_POINTER(NULL) /* patched later */' else: expr = db.get(value) if typeOf(value) is Void: Modified: pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py Tue Sep 11 19:09:45 2007 @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ - CompositeOffset, ArrayLengthOffset, WeakGcAddress, fakeweakaddress, \ + CompositeOffset, ArrayLengthOffset, \ GCHeaderOffset, WeakRef, fakeweakref from pypy.translator.c.support import cdecl @@ -112,15 +112,6 @@ else: return 'NULL' -def name_weakgcaddress(value, db): - assert isinstance(value, fakeweakaddress) - if value.ref is None: - return 'HIDE_POINTER(NULL)' - else: - ob = value.ref() - assert ob is not None - return 'HIDE_POINTER(%s)'%db.get(ob) - def name_weakref(value, db): assert isinstance(value, fakeweakref) target = value.get() @@ -142,7 +133,6 @@ Bool: name_bool, Void: name_void, Address: name_address, - WeakGcAddress: name_weakgcaddress, WeakRef: name_weakref, } @@ -157,7 +147,6 @@ Bool: 'bool_t @', Void: 'void @', Address: 'void* @', - WeakGcAddress: 'GC_hidden_pointer @', WeakRef: 'GCWeakRef @', } @@ -172,7 +161,6 @@ Bool: '0 /* error */', Void: '/* error */', Address: 'NULL', - WeakGcAddress: 'HIDE_POINTER(NULL)', WeakRef: 'NULL', } Modified: pypy/branch/cleanup-weakref/pypy/translator/c/src/address.h ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/src/address.h (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/src/address.h Tue Sep 11 19:09:45 2007 @@ -18,7 +18,6 @@ #define OP_CAST_ADR_TO_INT(x, r) r = ((long)x) #define OP_CAST_INT_TO_ADR(x, r) r = ((void *)(x)) -#define OP_CAST_WEAKADR_TO_INT(x, r) r = ((long)x) #ifndef HIDE_POINTER #define HIDE_POINTER(p) (p) Modified: pypy/branch/cleanup-weakref/pypy/translator/c/test/test_lladdresses.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/test/test_lladdresses.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/test/test_lladdresses.py Tue Sep 11 19:09:45 2007 @@ -120,44 +120,3 @@ return result fn = compile(f, [int]) assert fn(1) == 2 - -def test_weakaddress(): - from pypy.rlib.objectmodel import cast_object_to_weakgcaddress - from pypy.rlib.objectmodel import cast_weakgcaddress_to_object - from pypy.rpython.lltypesystem.lloperation import llop - class A(object): - pass - def func(i): - l1 = [] - l2 = [] - for i in range(i): - a = A() - l1.append(a) - l2.append(cast_object_to_weakgcaddress(a)) - return len(l1) == len(l2) - fn = compile(func, [int]) - assert fn(10) - -def test_constant_weakaddress(): - from pypy.rlib.objectmodel import cast_object_to_weakgcaddress - from pypy.rlib.objectmodel import cast_weakgcaddress_to_object - from pypy.rpython.lltypesystem.lloperation import llop - class A(object): - pass - constant_a = A() - constant_weak_a = cast_object_to_weakgcaddress(constant_a) - l = [constant_weak_a] - def func(i): - l1 = [] - l2 = [] - l3 = [] - l4 = [] - for i in range(i): - a = A() - l1.append(a) - l2.append(cast_object_to_weakgcaddress(a)) - l3.append(constant_a) - l4.extend(l) - return len(l1) == len(l2) == len(l3) == len(l4) - fn = compile(func, [int]) - assert fn(10) Modified: pypy/branch/cleanup-weakref/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/test/test_newgc.py Tue Sep 11 19:09:45 2007 @@ -8,7 +8,6 @@ from pypy.translator.c import genc, gc from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.objectmodel import cast_weakgcaddress_to_object, cast_object_to_weakgcaddress from pypy import conftest def compile_func(fn, inputtypes, t=None, gcpolicy="ref"): @@ -255,125 +254,6 @@ res = fn(112) assert res == -8 -class Weakrefable(object): - __lifeline__ = None - -class Weakref(object): - def __init__(self, lifeline, index, obj, callback): - self.address = cast_object_to_weakgcaddress(obj) - self.callback = callback - self.lifeline_addr = cast_object_to_weakgcaddress(lifeline) - self.index = index - - def ref(self): - return cast_weakgcaddress_to_object(self.address, Weakrefable) - - def invalidate(self): - self.address = llmemory.WEAKNULL - self.lifeline_addr = llmemory.WEAKNULL - if self.callback is not None: - self.callback(self) - - def __del__(self): - lifeline = cast_weakgcaddress_to_object(self.lifeline_addr, - WeakrefLifeline) - if lifeline is not None: - lifeline.ref_is_dead(self.index) - -class WeakrefLifeline(object): - def __init__(self, obj): - self.refs = [] - - def __del__(self): - i = 0 - while i < len(self.refs): - addr_ref = self.refs[i] - if cast_weakgcaddress_to_object(addr_ref, Weakref) is not None: - ref = cast_weakgcaddress_to_object(addr_ref, Weakref) - ref.invalidate() - i += 1 - - def get_weakref(self, obj, callback): - ref = Weakref(self, len(self.refs), obj, callback) - addr_ref = cast_object_to_weakgcaddress(ref) - self.refs.append(addr_ref) - return ref - - def ref_is_dead(self, index): - self.refs[index] = llmemory.WEAKNULL - -def get_weakref(obj, callback=None): - assert isinstance(obj, Weakrefable) - if obj.__lifeline__ is None: - obj.__lifeline__ = WeakrefLifeline(obj) - return obj.__lifeline__.get_weakref(obj, callback) - -def test_weakref_alive(): - def func(): - f = Weakrefable() - f.x = 32 - ref1 = get_weakref(f) - ref2 = get_weakref(f) - return f.x + ref2.ref().x - assert func() == 64 - f = compile_func(func, []) - assert f() == 64 - -def test_weakref_dying(): - def g(): - f = Weakrefable() - f.x = 32 - return get_weakref(f) - def func(): - ref = g() - return ref.ref() is None - assert func() - f = compile_func(func, []) - assert f() - -def test_weakref_callback(): - global_w = Weakrefable() - global_w.x = 31 - def g(ref): - global_w.x = 32 - def h(): - f = Weakrefable() - f.x = 32 - ref = get_weakref(f, g) - return ref - def func(): - ref = h() - return (ref.ref() is None and ref.callback is not None and - global_w.x == 32) - assert func() - f = compile_func(func, []) - assert f() - -def test_weakref_dont_always_callback(): - global_w1 = Weakrefable() - global_w1.x = 30 - global_w2 = Weakrefable() - global_w2.x = 30 - def g1(ref): - global_w1.x = 32 - def g2(ref): - global_w2.x = 32 - def h(): - f = Weakrefable() - f.x = 32 - ref1 = get_weakref(f, g1) - ref2 = get_weakref(f, g2) - if f.x % 2 == 0: # start a new block to make sure ref2 dies before f - ref2 = None - f.x = 12 - return ref1 - def func(): - ref = h() - return global_w1.x == 32 and global_w2.x == 30 - assert func() - f = compile_func(func, []) - assert f() - def test_gc_x_operations(): t = TranslationContext() from pypy.rlib.rgc import gc_clone, gc_swap_pool Modified: pypy/branch/cleanup-weakref/pypy/translator/cli/metavm.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/cli/metavm.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/cli/metavm.py Tue Sep 11 19:09:45 2007 @@ -120,13 +120,14 @@ '[mscorlib]System.Collections.Generic.IEqualityComparer`1)' % dict_type) -class _CastWeakAdrToPtr(MicroInstruction): - def render(self, generator, op): - RESULTTYPE = op.result.concretetype - resulttype = generator.cts.lltype_to_cts(RESULTTYPE) - generator.load(op.args[0]) - generator.ilasm.call_method('object class %s::get_Target()' % WEAKREF, True) - generator.ilasm.opcode('castclass', resulttype) +#XXX adapt to new way of things +#class _CastWeakAdrToPtr(MicroInstruction): +# def render(self, generator, op): +# RESULTTYPE = op.result.concretetype +# resulttype = generator.cts.lltype_to_cts(RESULTTYPE) +# generator.load(op.args[0]) +# generator.ilasm.call_method('object class %s::get_Target()' % WEAKREF, True) +# generator.ilasm.opcode('castclass', resulttype) class MapException(MicroInstruction): COUNT = 0 @@ -226,7 +227,7 @@ RuntimeNew = _RuntimeNew() OOString = _OOString() NewCustomDict = _NewCustomDict() -CastWeakAdrToPtr = _CastWeakAdrToPtr() +#CastWeakAdrToPtr = _CastWeakAdrToPtr() Box = _Box() Unbox = _Unbox() NewArray = _NewArray() Modified: pypy/branch/cleanup-weakref/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/cli/opcodes.py Tue Sep 11 19:09:45 2007 @@ -1,6 +1,6 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, OOString, DownCast, NewCustomDict,\ - CastWeakAdrToPtr, MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ + MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ TypeOf, CastPrimitive from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive @@ -60,7 +60,6 @@ 'indirect_call': [IndirectCall], 'cast_ptr_to_weakadr': [PushAllArgs, 'newobj instance void class %s::.ctor(object)' % WEAKREF], - 'cast_weakadr_to_ptr': [CastWeakAdrToPtr], 'gc__collect': 'call void class [mscorlib]System.GC::Collect()', 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, Modified: pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py Tue Sep 11 19:09:45 2007 @@ -24,7 +24,6 @@ lltype.UniChar: unichr(0xFFFF), # XXX is this always right? lltype.Bool: True, llmemory.Address: NULL, - llmemory.WeakGcAddress: llmemory.fakeweakaddress(None), llmemory.WeakRef: llmemory.fakeweakref(None), lltype.Void: None} Modified: pypy/branch/cleanup-weakref/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/jvm/metavm.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/jvm/metavm.py Tue Sep 11 19:09:45 2007 @@ -126,20 +126,21 @@ generator.emit(jvmgen.CUSTOMDICTMAKE) NewCustomDict = _NewCustomDict() -class _CastPtrToWeakAddress(MicroInstruction): - def render(self, generator, op): - arg = op.args[0] - generator.prepare_cast_ptr_to_weak_address() - generator.load(arg) - generator.finalize_cast_ptr_to_weak_address(arg.concretetype) - generator.store(op.result) -CastPtrToWeakAddress = _CastPtrToWeakAddress() +#XXX adapt to new way of things +#class _CastPtrToWeakAddress(MicroInstruction): +# def render(self, generator, op): +# arg = op.args[0] +# generator.prepare_cast_ptr_to_weak_address() +# generator.load(arg) +# generator.finalize_cast_ptr_to_weak_address(arg.concretetype) +# generator.store(op.result) +#CastPtrToWeakAddress = _CastPtrToWeakAddress() -class _CastWeakAddressToPtr(MicroInstruction): - def render(self, generator, op): - RESULTTYPE = op.result.concretetype - generator.cast_weak_address_to_ptr(RESULTTYPE) -CastWeakAddressToPtr = _CastWeakAddressToPtr() +#class _CastWeakAddressToPtr(MicroInstruction): +# def render(self, generator, op): +# RESULTTYPE = op.result.concretetype +# generator.cast_weak_address_to_ptr(RESULTTYPE) +#CastWeakAddressToPtr = _CastWeakAddressToPtr() CASTS = { Modified: pypy/branch/cleanup-weakref/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/jvm/opcodes.py Tue Sep 11 19:09:45 2007 @@ -10,7 +10,7 @@ SetField, GetField, DownCast, RuntimeNew, OOString, CastTo, PushPrimitive from pypy.translator.jvm.metavm import \ IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ - CastPtrToWeakAddress, CastWeakAddressToPtr, CastPrimitive + CastPrimitive from pypy.rpython.ootypesystem import ootype import pypy.translator.jvm.generator as jvmgen @@ -65,8 +65,6 @@ 'direct_call': [Call, StoreResult], 'indirect_call': [PushAllArgs, IndirectCall, StoreResult], - 'cast_ptr_to_weakadr': [CastPtrToWeakAddress], - 'cast_weakadr_to_ptr': CastWeakAddressToPtr, 'gc__collect': jvmgen.SYSTEMGC, 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, Modified: pypy/branch/cleanup-weakref/pypy/translator/llvm/test/test_lladdresses.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/llvm/test/test_lladdresses.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/llvm/test/test_lladdresses.py Tue Sep 11 19:09:45 2007 @@ -170,20 +170,3 @@ fn = compile_function(f, []) assert f() == 579 -def test_weakaddress(): - from pypy.rlib.objectmodel import cast_object_to_weakgcaddress - from pypy.rlib.objectmodel import cast_weakgcaddress_to_object - from pypy.rpython.lltypesystem.lloperation import llop - class A(object): - pass - def func(i): - l1 = [] - l2 = [] - for i in range(i): - a = A() - l1.append(a) - l2.append(cast_object_to_weakgcaddress(a)) - return len(l1) == len(l2) - fn = compile_function(func, [int]) - assert fn(10) - Modified: pypy/branch/cleanup-weakref/pypy/translator/oosupport/test_template/objectmodel.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/oosupport/test_template/objectmodel.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/oosupport/test_template/objectmodel.py Tue Sep 11 19:09:45 2007 @@ -1,8 +1,6 @@ import py from pypy.rlib.test.test_objectmodel import BaseTestObjectModel as RLibBase -from pypy.rlib.objectmodel import cast_object_to_weakgcaddress,\ - cast_weakgcaddress_to_object class BaseTestObjectModel(RLibBase): def test_rdict_of_void_copy(self): @@ -15,43 +13,3 @@ return len(d1) assert self.interpret(fn, []) == 2 - # this test is copied from TestLLtype in - # rlib/test_objectmodel.py. It is not in TestOOtype because at - # the moment llinterpret can't handle cast_*weakadr* - def test_cast_to_and_from_weakaddress(self): - class A(object): - pass - class B(object): - pass - def f(): - a = A() - addr = cast_object_to_weakgcaddress(a) - return a is cast_weakgcaddress_to_object(addr, A) - res = self.interpret(f, []) - assert res -## def g(): -## a = A() -## addr = cast_object_to_weakgcaddress(a) -## return cast_weakgcaddress_to_int(addr) -## assert isinstance(self.interpret(f, []), int) - - def test_weakref_const(self): - class A(object): - def __init__(self): - self.x = 42 - a = A() - weak = cast_object_to_weakgcaddress(a) - def f(): - a.x = 10 - b = cast_weakgcaddress_to_object(weak, A) - return b.x - assert self.interpret(f, []) == 10 - - def test_weakref_const_null(self): - class A(object): - pass - weak = cast_object_to_weakgcaddress(None) - def f(): - b = cast_weakgcaddress_to_object(weak, A) - return b - assert self.interpret(f, []) is None Modified: pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py Tue Sep 11 19:09:45 2007 @@ -296,7 +296,7 @@ self.retval_float = 0.0 self.retval_addr = llmemory.NULL self.retval_ref = frame.null_saved_ref - self.retval_weak = llmemory.WEAKNULL + self.retval_weak = llmemory.WEAKREFNULL self.exception = None self.masterarray = lltype.malloc(frame.FRAME_INFO_ARRAY, 0, immortal=True) Modified: pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py Tue Sep 11 19:09:45 2007 @@ -20,7 +20,7 @@ (lltype.SignedLongLong, 'longlong'), (lltype.Signed, 'long'), (lltype.Float, 'float'), - (llmemory.WeakGcAddress, 'weak'), + (llmemory.WeakRef, 'weak'), ] STORAGE_TYPES = [] @@ -55,8 +55,8 @@ return lltype.SignedLongLong elif T is llmemory.Address: return llmemory.Address - elif T is llmemory.WeakGcAddress: - return llmemory.WeakGcAddress + elif T is llmemory.WeakRef: + return llmemory.WeakRef elif isinstance(T, lltype.Primitive): return lltype.Signed else: Modified: pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py Tue Sep 11 19:09:45 2007 @@ -370,8 +370,8 @@ code.fetch_retval_addr, [], annmodel.SomeAddress()), SAVED_REFERENCE: mixlevelannotator.constfunc( code.fetch_retval_ref, [], annmodel.SomePtr(SAVED_REFERENCE)), - llmemory.WeakGcAddress: mixlevelannotator.constfunc( - code.fetch_retval_weak, [], annmodel.SomeWeakGcAddress()), + llmemory.WeakRef: mixlevelannotator.constfunc( + code.fetch_retval_weak, [], annmodel.SomeLLWeakRef()), } s_StatePtr = annmodel.SomePtr(frame.OPAQUE_STATE_HEADER_PTR) @@ -422,9 +422,9 @@ code.resume_after_ref, [s_StatePtr, annmodel.SomePtr(SAVED_REFERENCE)], annmodel.s_None), - llmemory.WeakGcAddress: mixlevelannotator.constfunc( + llmemory.WeakRef: mixlevelannotator.constfunc( code.resume_after_weak, - [s_StatePtr, annmodel.SomeWeakGcAddress()], + [s_StatePtr, annmodel.SomeLLWeakRef()], annmodel.s_None), } exception_def = bk.getuniqueclassdef(Exception) From cfbolz at codespeak.net Tue Sep 11 19:20:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Sep 2007 19:20:01 +0200 (CEST) Subject: [pypy-svn] r46479 - pypy/branch/cleanup-weakref/pypy/module/_weakref Message-ID: <20070911172001.806298107@code0.codespeak.net> Author: cfbolz Date: Tue Sep 11 19:20:00 2007 New Revision: 46479 Modified: pypy/branch/cleanup-weakref/pypy/module/_weakref/interp__weakref.py Log: forgotten import Modified: pypy/branch/cleanup-weakref/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/branch/cleanup-weakref/pypy/module/_weakref/interp__weakref.py Tue Sep 11 19:20:00 2007 @@ -4,7 +4,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.gateway import interp2app, ObjSpace -from pypy.rpython.lltypesystem.llmemory import WEAKNULL import weakref From cfbolz at codespeak.net Tue Sep 11 19:21:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Sep 2007 19:21:05 +0200 (CEST) Subject: [pypy-svn] r46480 - pypy/branch/cleanup-weakref/pypy/rpython/memory/test Message-ID: <20070911172105.B030B8107@code0.codespeak.net> Author: cfbolz Date: Tue Sep 11 19:21:05 2007 New Revision: 46480 Modified: pypy/branch/cleanup-weakref/pypy/rpython/memory/test/test_address.py Log: another one Modified: pypy/branch/cleanup-weakref/pypy/rpython/memory/test/test_address.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/memory/test/test_address.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/memory/test/test_address.py Tue Sep 11 19:21:05 2007 @@ -9,7 +9,7 @@ from pypy.rpython.memory.lladdress import _address, NULL from pypy.rpython.memory.lladdress import raw_malloc, raw_free, raw_memcopy from pypy.rpython.memory.lladdress import get_py_object, get_address_of_object -from pypy.rpython.lltypesystem.llmemory import Address, NullAddressError, WEAKNULL +from pypy.rpython.lltypesystem.llmemory import Address, NullAddressError from pypy.rpython.memory.simulator import MemorySimulatorError from pypy.rpython.memory.test.test_llinterpsim import interpret from pypy.rpython.lltypesystem import lltype From antocuni at codespeak.net Tue Sep 11 19:35:19 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 11 Sep 2007 19:35:19 +0200 (CEST) Subject: [pypy-svn] r46481 - in pypy/dist/pypy/translator: . cli cli/test jvm/test oosupport oosupport/test_template Message-ID: <20070911173519.C29F08107@code0.codespeak.net> Author: antocuni Date: Tue Sep 11 19:35:19 2007 New Revision: 46481 Added: pypy/dist/pypy/translator/oosupport/support.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/support.py pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/driver.py pypy/dist/pypy/translator/jvm/test/runtest.py pypy/dist/pypy/translator/jvm/test/test_builtin.py pypy/dist/pypy/translator/oosupport/test_template/builtin.py Log: use the NT os flags also for genjvm; move the patching code to oosupport/ Modified: pypy/dist/pypy/translator/cli/support.py ============================================================================== --- pypy/dist/pypy/translator/cli/support.py (original) +++ pypy/dist/pypy/translator/cli/support.py Tue Sep 11 19:35:19 2007 @@ -66,40 +66,3 @@ f.write('%s: %d\n' % (label, self.counters[key])) f.close() -NT_OS = dict( - O_RDONLY = 0x0000, - O_WRONLY = 0x0001, - O_RDWR = 0x0002, - O_APPEND = 0x0008, - O_CREAT = 0x0100, - O_TRUNC = 0x0200, - O_TEXT = 0x4000, - O_BINARY = 0x8000 - ) - -def _patch_os(defs=None): - """ - Modify the value of some attributes of the os module to be sure - they are the same on every platform pypy is compiled on. Returns a - dictionary containing the original values that can be passed to - patch_os to rollback to the original values. - """ - - import os - if defs is None: - defs = NT_OS - olddefs = {} - for name, value in defs.iteritems(): - try: - olddefs[name] = getattr(os, name) - except AttributeError: - pass - setattr(os, name, value) - return olddefs - -def patch(): - olddefs = _patch_os() - return olddefs, - -def unpatch(olddefs): - _patch_os(olddefs) Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Tue Sep 11 19:35:19 2007 @@ -21,7 +21,7 @@ from pypy.translator.cli.database import LowLevelDatabase from pypy.translator.cli.sdk import SDK from pypy.translator.cli.entrypoint import BaseEntryPoint -from pypy.translator.cli.support import patch, unpatch +from pypy.translator.oosupport.support import patch_os, unpatch_os FLOAT_PRECISION = 8 @@ -132,11 +132,11 @@ def compile_function(func, annotation=[], graph=None, backendopt=True, auto_raise_exc=False, exctrans=False): - olddefs = patch() + olddefs = patch_os() gen = _build_gen(func, annotation, graph, backendopt, exctrans) gen.generate_source() exe_name = gen.build_exe() - unpatch(*olddefs) # restore original values + unpatch_os(olddefs) # restore original values return CliFunctionWrapper(exe_name, func.__name__, auto_raise_exc) def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False): Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Tue Sep 11 19:35:19 2007 @@ -647,13 +647,13 @@ 'Generating CLI source') def task_compile_cli(self): - from pypy.translator.cli.support import unpatch + from pypy.translator.oosupport.support import unpatch_os from pypy.translator.cli.test.runtest import CliFunctionWrapper filename = self.gen.build_exe() self.c_entryp = CliFunctionWrapper(filename) # restore original os values if hasattr(self, 'old_cli_defs'): - unpatch(*self.old_cli_defs) + unpatch_os(self.old_cli_defs) self.log.info("Compiled %s" % filename) if self.standalone and self.exe_name: @@ -724,8 +724,13 @@ 'Generating JVM source') def task_compile_jvm(self): + from pypy.translator.oosupport.support import unpatch_os self.jvmsource.compile() self.c_entryp = lambda *args: eval(self.jvmsource.execute(args)) + # restore original os values + if hasattr(self, 'old_cli_defs'): + unpatch_os(self.old_cli_defs) + self.log.info("Compiled JVM source") task_compile_jvm = taskdef(task_compile_jvm, ['source_jvm'], 'Compiling JVM source') @@ -760,9 +765,9 @@ # patch some attributes of the os module to make sure they # have the same value on every platform. backend, ts = driver.get_backend_and_type_system() - if backend == 'cli': - from pypy.translator.cli.support import patch - driver.old_cli_defs = patch() + if backend in ('cli', 'jvm'): + from pypy.translator.oosupport.support import patch_os + driver.old_cli_defs = patch_os() target = targetspec_dic['target'] spec = target(driver, args) Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Tue Sep 11 19:35:19 2007 @@ -9,6 +9,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.annotation.model import lltype_to_annotation from pypy.translator.translator import TranslationContext +from pypy.translator.oosupport.support import patch_os, unpatch_os from pypy.translator.jvm.genjvm import \ generate_source_for_function, JvmError, detect_missing_support_programs from pypy.translator.jvm.option import getoption @@ -91,7 +92,9 @@ else: self._func = fn self._ann = ann + olddefs = patch_os() self._jvm_src = generate_source_for_function(fn, ann) + unpatch_os(olddefs) if not getoption('noasm'): self._jvm_src.compile() return JvmGeneratedSourceWrapper(self._jvm_src) Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Tue Sep 11 19:35:19 2007 @@ -5,9 +5,6 @@ class TestJavaBuiltin(JvmTest, BaseTestBuiltin): - def test_os_flags(self): - py.test.skip('fixme!') - def test_os_open_write(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") Added: pypy/dist/pypy/translator/oosupport/support.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/support.py Tue Sep 11 19:35:19 2007 @@ -0,0 +1,36 @@ +NT_OS = dict( + O_RDONLY = 0x0000, + O_WRONLY = 0x0001, + O_RDWR = 0x0002, + O_APPEND = 0x0008, + O_CREAT = 0x0100, + O_TRUNC = 0x0200, + O_TEXT = 0x4000, + O_BINARY = 0x8000 + ) + +def _patch_os(defs=None): + """ + Modify the value of some attributes of the os module to be sure + they are the same on every platform pypy is compiled on. Returns a + dictionary containing the original values that can be passed to + patch_os to rollback to the original values. + """ + + import os + if defs is None: + defs = NT_OS + olddefs = {} + for name, value in defs.iteritems(): + try: + olddefs[name] = getattr(os, name) + except AttributeError: + pass + setattr(os, name, value) + return olddefs + +def patch_os(): + return _patch_os() + +def unpatch_os(olddefs): + _patch_os(olddefs) Modified: pypy/dist/pypy/translator/oosupport/test_template/builtin.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/builtin.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/builtin.py Tue Sep 11 19:35:19 2007 @@ -9,7 +9,7 @@ class BaseTestBuiltin(BaseTestRbuiltin): def test_os_flags(self): - from pypy.translator.cli.support import NT_OS + from pypy.translator.oosupport.support import NT_OS def fn(): return os.O_CREAT assert self.interpret(fn, []) == NT_OS['O_CREAT'] From arigo at codespeak.net Tue Sep 11 21:13:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 21:13:40 +0200 (CEST) Subject: [pypy-svn] r46482 - in pypy/branch/cleanup-weakref/pypy: annotation rpython rpython/lltypesystem rpython/memory/gctransform translator translator/c translator/c/test translator/stackless Message-ID: <20070911191340.4136980EC@code0.codespeak.net> Author: arigo Date: Tue Sep 11 21:13:38 2007 New Revision: 46482 Modified: pypy/branch/cleanup-weakref/pypy/annotation/builtin.py pypy/branch/cleanup-weakref/pypy/annotation/model.py pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lltype.py pypy/branch/cleanup-weakref/pypy/rpython/memory/gctransform/boehm.py pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py pypy/branch/cleanup-weakref/pypy/rpython/rweakref.py pypy/branch/cleanup-weakref/pypy/translator/c/database.py pypy/branch/cleanup-weakref/pypy/translator/c/gc.py pypy/branch/cleanup-weakref/pypy/translator/c/node.py pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py pypy/branch/cleanup-weakref/pypy/translator/c/test/test_boehm.py pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py Log: Turn WeakRef into a container type. This allows a number of simplifications at various places. Thanks pedronis. Modified: pypy/branch/cleanup-weakref/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/annotation/builtin.py (original) +++ pypy/branch/cleanup-weakref/pypy/annotation/builtin.py Tue Sep 11 21:13:38 2007 @@ -9,7 +9,7 @@ from pypy.annotation.model import SomeFloat, unionof from pypy.annotation.model import SomePBC, SomeInstance, SomeDict from pypy.annotation.model import SomeExternalObject -from pypy.annotation.model import SomeWeakRef, SomeLLWeakRef +from pypy.annotation.model import SomeWeakRef from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata from pypy.annotation.model import s_ImpossibleValue @@ -577,7 +577,7 @@ s_obj.ll_ptrtype.TO._gckind != 'gc'): raise Exception("bad type for argument to weakref_create(): %r" % ( s_obj,)) - return SomeLLWeakRef() + return SomePtr(llmemory.WeakRefPtr) def llweakref_deref(s_ptrtype, s_wref): if not (s_ptrtype.is_constant() and @@ -585,8 +585,9 @@ s_ptrtype.const.TO._gckind == 'gc'): raise Exception("weakref_deref() arg 1 must be a constant " "ptr type, got %s" % (s_ptrtype,)) - if not isinstance(s_wref, SomeLLWeakRef): - raise Exception("weakref_deref() arg 2 must be a llweakref, " + if not (isinstance(s_wref, SomePtr) and + s_wref.ll_ptrtype == llmemory.WeakRefPtr): + raise Exception("weakref_deref() arg 2 must be a WeakRefPtr, " "got %s" % (s_wref,)) return SomePtr(s_ptrtype.const) Modified: pypy/branch/cleanup-weakref/pypy/annotation/model.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/annotation/model.py (original) +++ pypy/branch/cleanup-weakref/pypy/annotation/model.py Tue Sep 11 21:13:38 2007 @@ -510,12 +510,6 @@ def can_be_none(self): return False -class SomeLLWeakRef(SomeObject): - immutable = True - - def can_be_none(self): - return False - # ____________________________________________________________ # memory addresses @@ -593,7 +587,6 @@ (SomeChar(), lltype.Char), (SomeUnicodeCodePoint(), lltype.UniChar), (SomeAddress(), llmemory.Address), - (SomeLLWeakRef(), llmemory.WeakRef), ] def annotation_to_lltype(s_val, info=None): Modified: pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/llmemory.py Tue Sep 11 21:13:38 2007 @@ -487,75 +487,66 @@ def cast_int_to_adr(int): raise NotImplementedError("cast_int_to_adr") - # ____________________________________________________________ +# Weakrefs. # -# WeakRef - like in RPython, they are pointers that don't keep their -# target malloc'ed structure alive. When the target dies -# all WeakRefs to it are cleared. +# An object of type WeakRef is a small GC-managed object that contains +# a weak reference to another GC-managed object, as in regular Python. # -# ____________________________________________________________ - -class fakeweakref(object): - # XXX convoluted code to support both lltype._ptr and simulatorptr - def __init__(self, ob): - if ob is not None: - if isinstance(ob, lltype._ptr): - ob = lltype.normalizeptr(ob)._obj - self.obref = weakref.ref(ob) - self.ref = self.get # backward compatibility - # umpf - from pypy.rpython.memory import lltypesimulation - if isinstance(ob, (lltype._ptr,lltypesimulation.simulatorptr)): - self.id = ob._cast_to_int() - else: - self.id = id(ob) - else: - self.obref = None - self.ref = None - def get(self): - if self.obref is None: - return None - ob = self.obref() - # xxx stop-gap - #if ob is None: - # raise DanglingPointerError - if isinstance(ob, lltype._container): - ob = ob._as_ptr() - return ob - def __repr__(self): - if self.obref is None: - s = 'NULL' - else: - s = str(self.obref) - return '<%s %s>' % (self.__class__.__name__, s) - #def cast_to_int(self): - # # this is not always the behaviour that is really happening - # # but make sure that nobody depends on it - # return self.id ^ ~3 - - -WeakRef = lltype.Primitive("WeakRef", fakeweakref(None)) - -def weakref_create(obj): - PTRTYPE = lltype.typeOf(obj) +class _WeakRefType(lltype.ContainerType): + _gckind = 'gc' + def __str__(self): + return "WeakRef" + +WeakRef = _WeakRefType() +WeakRefPtr = lltype.Ptr(WeakRef) + +def weakref_create(ptarget): + # ptarget should not be a nullptr + PTRTYPE = lltype.typeOf(ptarget) assert isinstance(PTRTYPE, lltype.Ptr) assert PTRTYPE.TO._gckind == 'gc' - return fakeweakref(obj) + assert ptarget + return _wref(ptarget)._as_ptr() -def weakref_deref(PTRTYPE, wref): +def weakref_deref(PTRTYPE, pwref): + # pwref should not be a nullptr assert isinstance(PTRTYPE, lltype.Ptr) assert PTRTYPE.TO._gckind == 'gc' - assert lltype.typeOf(wref) == WeakRef - p = wref.get() + assert lltype.typeOf(pwref) == WeakRefPtr + p = pwref._obj._dereference() if p is None: return lltype.nullptr(PTRTYPE.TO) else: return lltype.cast_pointer(PTRTYPE, p) -fakeweakref._TYPE = WeakRef -WEAKREFNULL = fakeweakref(None) +class _wref(lltype._container): + _gckind = 'gc' + _TYPE = WeakRef + + def __init__(self, ptarget): + if ptarget is None: + self._obref = lambda: None + else: + obj = lltype.normalizeptr(ptarget)._obj + self._obref = weakref.ref(obj) + + def _dereference(self): + obj = self._obref() + if obj is None: + return None + else: + return obj._as_ptr() + + def __repr__(self): + return '<%s>' % (self,) + + def __str__(self): + return 'wref -> %s' % (self._obref(),) + +# a prebuilt pointer to a dead low-level weakref +dead_wref = _wref(None)._as_ptr() # ____________________________________________________________ Modified: pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lloperation.py Tue Sep 11 21:13:38 2007 @@ -91,6 +91,9 @@ (self in (llop.getfield, llop.getarrayitem) and ARGTYPES[0].TO._hints.get('immutable'))) + def __repr__(self): + return '' % (getattr(self, 'opname', '?'),) + def enum_ops_without_sideeffects(raising_is_ok=False): """Enumerate operations that have no side-effects Modified: pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/lltype.py Tue Sep 11 21:13:38 2007 @@ -161,7 +161,10 @@ def _nofield(self, name): raise AttributeError("no field %r" % name) - + + def _container_example(self): + raise NotImplementedError + class Struct(ContainerType): _gckind = 'raw' Modified: pypy/branch/cleanup-weakref/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/memory/gctransform/boehm.py Tue Sep 11 21:13:38 2007 @@ -43,10 +43,10 @@ self.malloc_varsize_ptr = self.inittime_helper( ll_malloc_varsize, [lltype.Signed]*4, llmemory.Address, inline=False) self.weakref_create_ptr = self.inittime_helper( - ll_weakref_create, [llmemory.Address], llmemory.WeakRef, + ll_weakref_create, [llmemory.Address], llmemory.WeakRefPtr, inline=False) self.weakref_deref_ptr = self.inittime_helper( - ll_weakref_deref, [llmemory.WeakRef], llmemory.Address) + ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address) self.mixlevelannotator.finish() # for now self.mixlevelannotator.backend_optimize() @@ -137,7 +137,7 @@ resulttype=llmemory.Address) v_wref = hop.genop("direct_call", [self.weakref_create_ptr, v_addr], - resulttype=llmemory.WeakRef) + resulttype=llmemory.WeakRefPtr) hop.cast_result(v_wref) def gct_weakref_deref(self, hop): @@ -149,34 +149,33 @@ ########## weakrefs ########## -# Boehm: we implement weakrefs with an extra indirection: GCWeakRef is a -# pointer to malloced data containing only a Boehm disappearing link. -# This allows the disappearing link to remain at a fixed address. -# We also don't have to hide the link's value with HIDE_POINTER(), -# because we use GC_MALLOC_ATOMIC(). - -# Declared in gc.py: typedef GC_PTR *GCWeakRef; +# Boehm: weakref objects are small structures containing only a Boehm +# disappearing link. We don't have to hide the link's value with +# HIDE_POINTER(), because we explicitly use GC_MALLOC_ATOMIC(). WEAKLINK = lltype.FixedSizeArray(llmemory.Address, 1) sizeof_weakreflink = llmemory.sizeof(WEAKLINK) +empty_weaklink = lltype.malloc(WEAKLINK, immortal=True) +empty_weaklink[0] = llmemory.NULL def ll_weakref_create(targetaddr): link = llop.boehm_malloc_atomic(llmemory.Address, sizeof_weakreflink) link.address[0] = targetaddr llop.boehm_disappearing_link(lltype.Void, link, targetaddr) - # abuse of llop.cast_pointer() - return llop.cast_pointer(llmemory.WeakRef, link) + return llmemory.cast_adr_to_ptr(link, llmemory.WeakRefPtr) def ll_weakref_deref(wref): - # abuse of llop.cast_pointer() - link = llop.cast_pointer(llmemory.Address, wref) + link = llmemory.cast_ptr_to_adr(wref) return link and link.address[0] -def convert_prebuilt_weakref_to(targetptr): +def convert_weakref_to(targetptr): # Prebuilt weakrefs don't really need to be weak at all, # but we need to emulate the structure expected by ll_weakref_deref(). # This is essentially the same code as in ll_weakref_create(), but I'm # not sure trying to share it is worth the hassle... - link = lltype.malloc(WEAKLINK, immortal=True) - link[0] = llmemory.cast_ptr_to_adr(targetptr) - return link + if not targetptr: + return empty_weaklink + else: + link = lltype.malloc(WEAKLINK, immortal=True) + link[0] = llmemory.cast_ptr_to_adr(targetptr) + return link Modified: pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py Tue Sep 11 21:13:38 2007 @@ -602,10 +602,11 @@ # Note: this code also works for the RPython-level calls 'weakref.ref(x)'. vlist = hop.inputargs(hop.args_r[0]) hop.exception_cannot_occur() - return hop.genop('weakref_create', vlist, resulttype=llmemory.WeakRef) + return hop.genop('weakref_create', vlist, resulttype=llmemory.WeakRefPtr) def rtype_weakref_deref(hop): - c_ptrtype, v_wref = hop.inputargs(lltype.Void, llmemory.WeakRef) + c_ptrtype, v_wref = hop.inputargs(lltype.Void, hop.args_r[1]) + assert v_wref.concretetype == llmemory.WeakRefPtr hop.exception_cannot_occur() return hop.genop('weakref_deref', [v_wref], resulttype=c_ptrtype.value) Modified: pypy/branch/cleanup-weakref/pypy/rpython/rweakref.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/rpython/rweakref.py (original) +++ pypy/branch/cleanup-weakref/pypy/rpython/rweakref.py Tue Sep 11 21:13:38 2007 @@ -6,19 +6,9 @@ from pypy.rpython.rclass import getinstancerepr from pypy.rpython.lltypesystem import lltype, llmemory - -class __extend__(annmodel.SomeLLWeakRef): - def rtyper_makerepr(self, rtyper): - return LLWeakRefRepr() - def rtyper_makekey(self): - return self.__class__, - -class LLWeakRefRepr(Repr): - lowleveltype = llmemory.WeakRef - # ____________________________________________________________ # -# RPython-level weakrefs +# RTyping of RPython-level weakrefs class __extend__(annmodel.SomeWeakRef): def rtyper_makerepr(self, rtyper): @@ -28,7 +18,7 @@ class WeakRefRepr(Repr): - lowleveltype = llmemory.WeakRef + lowleveltype = llmemory.WeakRefPtr def __init__(self, rtyper): self.rtyper = rtyper @@ -48,7 +38,7 @@ # obscure! if the annotator hasn't seen this object before, # we don't want to look at it now (confusion tends to result). if instance is None or not bk.have_seen(instance): - return llmemory.WeakRef._defl() + return llmemory.dead_wref else: repr = self.rtyper.bindingrepr(Constant(instance)) llinstance = repr.convert_const(instance) Modified: pypy/branch/cleanup-weakref/pypy/translator/c/database.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/database.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/database.py Tue Sep 11 21:13:38 2007 @@ -3,7 +3,7 @@ Struct, Array, FuncType, PyObject, Void, \ ContainerType, OpaqueType, FixedSizeArray, _uninitialized from pypy.rpython.lltypesystem import lltype -from pypy.rpython.lltypesystem.llmemory import Address +from pypy.rpython.lltypesystem.llmemory import Address, WeakRef, _WeakRefType from pypy.rpython.lltypesystem.rffi import CConstant from pypy.tool.sourcetools import valid_identifier from pypy.translator.c.primitive import PrimitiveName, PrimitiveType @@ -17,6 +17,8 @@ from pypy import conftest from pypy.translator.c import gc +class NoCorrespondingNode(Exception): + pass # ____________________________________________________________ @@ -84,8 +86,11 @@ node = ArrayDefNode(self, T, varlength) elif isinstance(T, OpaqueType) and hasattr(T, '_exttypeinfo'): node = ExtTypeOpaqueDefNode(self, T) + elif T == WeakRef: + REALT = self.gcpolicy.get_real_weakref_type() + node = self.gettypedefnode(REALT) else: - raise Exception("don't know about %r" % (T,)) + raise NoCorrespondingNode("don't know about %r" % (T,)) self.structdefnodes[key] = node self.pendingsetupnodes.append(node) return node @@ -94,14 +99,16 @@ if isinstance(T, Primitive): return PrimitiveType[T] elif isinstance(T, Ptr): - if isinstance(T.TO, FixedSizeArray): - # /me blames C + try: node = self.gettypedefnode(T.TO) - return node.getptrtype() + except NoCorrespondingNode: + pass else: - typename = self.gettype(T.TO) # who_asks not propagated - return typename.replace('@', '*@') - elif isinstance(T, (Struct, Array)): + if hasattr(node, 'getptrtype'): + return node.getptrtype() # special-casing because of C + typename = self.gettype(T.TO) # who_asks not propagated + return typename.replace('@', '*@') + elif isinstance(T, (Struct, Array, _WeakRefType)): node = self.gettypedefnode(T, varlength=varlength) if who_asks is not None: who_asks.dependencies[node] = True Modified: pypy/branch/cleanup-weakref/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/gc.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/gc.py Tue Sep 11 21:13:38 2007 @@ -175,10 +175,6 @@ class BoehmGcPolicy(BasicGcPolicy): transformerclass = boehm.BoehmGCTransformer - def __init__(self, *args, **kwds): - BasicGcPolicy.__init__(self, *args, **kwds) - self.weakref_llwrapper_cache = {} - def array_setup(self, arraydefnode): pass @@ -208,7 +204,7 @@ yield '#define USING_BOEHM_GC' def pre_gc_code(self): - return ['typedef GC_PTR *GCWeakRef;'] + return [] def gc_startup_code(self): if sys.platform == 'win32': @@ -217,16 +213,11 @@ yield 'GC_all_interior_pointers = 0;' yield 'GC_init();' - def name_weakref_to(self, target): - # the cache is essential to ensure that repeated calls to - # db.get(weakref) don't return new llwrapper structures all - # the time, which defeats the db.complete() logic. - try: - llwrapper = self.weakref_llwrapper_cache[target._obj] - except KeyError: - llwrapper = boehm.convert_prebuilt_weakref_to(target) - self.weakref_llwrapper_cache[target._obj] = llwrapper - return '((GCWeakRef)%s)' % (self.db.get(llwrapper),) + def get_real_weakref_type(self): + return boehm.WEAKLINK + + def convert_weakref_to(self, ptarget): + return boehm.convert_weakref_to(ptarget) def OP_GC_FETCH_EXCEPTION(self, funcgen, op): result = funcgen.expr(op.result) Modified: pypy/branch/cleanup-weakref/pypy/translator/c/node.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/node.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/node.py Tue Sep 11 21:13:38 2007 @@ -4,6 +4,7 @@ GcStruct, GcArray, RttiStruct, PyStruct, ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray, _pyobjheader +from pypy.rpython.lltypesystem import llmemory from pypy.translator.c.funcgen import FunctionCodeGenerator from pypy.translator.c.external import CExternalFunctionCodeGenerator from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring @@ -859,6 +860,16 @@ return PyObjectNode(db, T, obj) +def weakrefnode_factory(db, T, obj): + assert isinstance(obj, llmemory._wref) + ptarget = obj._dereference() + wrapper = db.gcpolicy.convert_weakref_to(ptarget) + container = wrapper._obj + T = typeOf(container) + nodefactory = ContainerNodeFactory[T.__class__] + return nodefactory(db, T, container) + + ContainerNodeFactory = { Struct: StructNode, GcStruct: StructNode, @@ -869,4 +880,5 @@ FuncType: FuncNode, OpaqueType: opaquenode_factory, PyObjectType: objectnode_factory, + llmemory._WeakRefType: weakrefnode_factory, } Modified: pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/primitive.py Tue Sep 11 21:13:38 2007 @@ -6,7 +6,7 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ - GCHeaderOffset, WeakRef, fakeweakref + GCHeaderOffset from pypy.translator.c.support import cdecl # ____________________________________________________________ @@ -112,14 +112,6 @@ else: return 'NULL' -def name_weakref(value, db): - assert isinstance(value, fakeweakref) - target = value.get() - if target is None: - return 'NULL' - else: - return db.gcpolicy.name_weakref_to(target) - # On 64 bit machines, SignedLongLong and Signed are the same, so the # order matters, because we want the Signed implementation. PrimitiveName = { @@ -133,7 +125,6 @@ Bool: name_bool, Void: name_void, Address: name_address, - WeakRef: name_weakref, } PrimitiveType = { @@ -147,7 +138,6 @@ Bool: 'bool_t @', Void: 'void @', Address: 'void* @', - WeakRef: 'GCWeakRef @', } PrimitiveErrorValue = { @@ -161,7 +151,6 @@ Bool: '0 /* error */', Void: '/* error */', Address: 'NULL', - WeakRef: 'NULL', } def define_c_primitive(ll_type, c_name): Modified: pypy/branch/cleanup-weakref/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/c/test/test_boehm.py Tue Sep 11 21:13:38 2007 @@ -146,38 +146,6 @@ assert 0 < res1 <= 10 assert 0 < res2 <= 5 - def test_weakgcaddress_is_weak(self): - py.test.skip("weakgcaddress as we know it is fragile") - from pypy.rpython.lltypesystem.lloperation import llop - from pypy.rlib.objectmodel import cast_object_to_weakgcaddress - class State: - pass - s = State() - class A(object): - def __del__(self): - s.a_dels += 1 - def f(i): - if i: - s.a_dels = 0 - a = A() - # this should not keep a alive - s.a = cast_object_to_weakgcaddress(a) - a = None - llop.gc__collect(lltype.Void) - llop.gc__collect(lltype.Void) - llop.gc__collect(lltype.Void) - return s.a_dels - fn = self.getcompiled(f, [int]) - # we can't demand that boehm has collected all of the objects, - # even with the gc__collect call. calling the compiled - # function twice seems to help, though. - fn(1) - fn(0) - fn(0) - res = fn(0) - print res - assert res == 1 - def test_del_raises(self): from pypy.rpython.lltypesystem.lloperation import llop import os Modified: pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py Tue Sep 11 21:13:38 2007 @@ -24,7 +24,6 @@ lltype.UniChar: unichr(0xFFFF), # XXX is this always right? lltype.Bool: True, llmemory.Address: NULL, - llmemory.WeakRef: llmemory.fakeweakref(None), lltype.Void: None} def error_value(T): Modified: pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/stackless/code.py Tue Sep 11 21:13:38 2007 @@ -296,7 +296,6 @@ self.retval_float = 0.0 self.retval_addr = llmemory.NULL self.retval_ref = frame.null_saved_ref - self.retval_weak = llmemory.WEAKREFNULL self.exception = None self.masterarray = lltype.malloc(frame.FRAME_INFO_ARRAY, 0, immortal=True) @@ -339,9 +338,6 @@ elif retval_code == frame.RETVAL_LONGLONG: global_state.retval_longlong = ( call_function_retval_longlong(fn, signature_index)) - elif retval_code == frame.RETVAL_WEAK: - global_state.retval_weak = ( - call_function_retval_weak(fn, signature_index)) else: assert False call_function.stackless_explicit = True @@ -449,12 +445,3 @@ global_state.retval_ref = frame.null_saved_ref return res fetch_retval_ref.stackless_explicit = True - -def fetch_retval_weak(): - e = global_state.exception - if e: - global_state.exception = None - raise e - else: - return global_state.retval_weak -fetch_retval_weak.stackless_explicit = True Modified: pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/stackless/frame.py Tue Sep 11 21:13:38 2007 @@ -20,7 +20,6 @@ (lltype.SignedLongLong, 'longlong'), (lltype.Signed, 'long'), (lltype.Float, 'float'), - (llmemory.WeakRef, 'weak'), ] STORAGE_TYPES = [] @@ -55,8 +54,6 @@ return lltype.SignedLongLong elif T is llmemory.Address: return llmemory.Address - elif T is llmemory.WeakRef: - return llmemory.WeakRef elif isinstance(T, lltype.Primitive): return lltype.Signed else: Modified: pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py (original) +++ pypy/branch/cleanup-weakref/pypy/translator/stackless/transform.py Tue Sep 11 21:13:38 2007 @@ -370,8 +370,6 @@ code.fetch_retval_addr, [], annmodel.SomeAddress()), SAVED_REFERENCE: mixlevelannotator.constfunc( code.fetch_retval_ref, [], annmodel.SomePtr(SAVED_REFERENCE)), - llmemory.WeakRef: mixlevelannotator.constfunc( - code.fetch_retval_weak, [], annmodel.SomeLLWeakRef()), } s_StatePtr = annmodel.SomePtr(frame.OPAQUE_STATE_HEADER_PTR) @@ -422,10 +420,6 @@ code.resume_after_ref, [s_StatePtr, annmodel.SomePtr(SAVED_REFERENCE)], annmodel.s_None), - llmemory.WeakRef: mixlevelannotator.constfunc( - code.resume_after_weak, - [s_StatePtr, annmodel.SomeLLWeakRef()], - annmodel.s_None), } exception_def = bk.getuniqueclassdef(Exception) self.resume_after_raising_ptr = mixlevelannotator.constfunc( From simonb at codespeak.net Tue Sep 11 21:30:49 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Tue, 11 Sep 2007 21:30:49 +0200 (CEST) Subject: [pypy-svn] r46483 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070911193049.C7C6880EC@code0.codespeak.net> Author: simonb Date: Tue Sep 11 21:30:48 2007 New Revision: 46483 Modified: pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: array coercion working now Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Tue Sep 11 21:30:48 2007 @@ -1,5 +1,6 @@ from pypy.rpython.error import TyperError -from pypy.annotation.model import SomeObject, SomeInteger +from pypy.annotation.model import lltype_to_annotation, SomeObject, SomeInteger +from pypy.rpython.numpy import aarray from pypy.rpython.numpy.aarray import SomeArray from pypy.annotation.pairtype import pairtype, pair from pypy.rlib.unroll import unrolling_iterable @@ -17,7 +18,8 @@ from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rpython.lltypesystem.lltype import \ GcArray, GcStruct, Number, Primitive, Signed, Ptr, Unsigned, Void, FixedSizeArray, Bool,\ - GcForwardReference, malloc, direct_arrayitems, direct_ptradd, nullptr, typeMethod + GcForwardReference, malloc, direct_arrayitems, direct_ptradd, nullptr, typeMethod,\ + cast_primitive from pypy.rpython.lltypesystem.rffi import cast from pypy.rpython.lltypesystem.rtuple import TupleRepr @@ -66,7 +68,7 @@ NPY_INTP = Signed # XX index type (see Py_intptr_t) -def ARRAY_ITER(ARRAY, INDEXARRAY): +def ARRAY_ITER(ARRAY, INDEXARRAY, ITEM): ndim = dim_of_ARRAY(ARRAY) unroll_ndim = unrolling_iterable(range(ndim)) @@ -89,9 +91,10 @@ it.dataptr = direct_ptradd(it.dataptr, -it.backstrides[i]) ll_iter_next._always_inline_ = True + DATA_PTR = Ptr(FixedSizeArray(ITEM, 1)) + ADTIIter = ADTInterface(None, { -# 'll_new':['SELF', ARRAY, ARRAY], - 'll_reset': (['self', ARRAY.TO.dataptr], Void), + 'll_reset': (['self', DATA_PTR], Void), 'll_next': (['self'], Void), }) ITER = Ptr( @@ -105,13 +108,14 @@ ("backstrides", INDEXARRAY), #("factors", INDEXARRAY), #("ao", ARRAY), # not needed.. - ("dataptr", ARRAY.TO.dataptr), # pointer to current item + ("dataptr", DATA_PTR), # pointer to current item #("contiguous", Bool), adtmeths = ADTIIter({ 'll_next': ll_iter_next, 'll_reset': ll_iter_reset, }), )) + return ITER def ll_mul_list(items, n): @@ -121,9 +125,13 @@ n -= 1 return result +_gen_iter_funcs_cache = {} def gen_iter_funcs(ndim): - unroll_ndim = unrolling_iterable(range(ndim)) + funcs = _gen_iter_funcs_cache.get(ndim) + if funcs: + return funcs + unroll_ndim = unrolling_iterable(range(ndim)) def ll_iter_new(ITER, ao, target_ao, iter_broadcast_to_shape): assert ao.dataptr # Suffix of ao.shape must match target_ao.shape @@ -140,6 +148,7 @@ i += 1 if broadcast: return iter_broadcast_to_shape(ITER, ao, target_ao) + debug_assert(ao.ndim == ndim, "ao.ndim == ndim") it = malloc(ITER) it.nd_m1 = ndim - 1 it.size = ll_mul_list(ao.shape, ndim) @@ -153,6 +162,7 @@ it.ll_reset(ao.dataptr) return it ll_iter_new._always_inline_ = True + #ll_iter_new._annspecialcase_ = "specialize:argtype(1, 2)" def ll_iter_broadcast_to_shape(ITER, ao, target_ao): "iterate over but broadcast to the shape of " @@ -181,22 +191,17 @@ it.ll_reset(ao.dataptr) return it ll_iter_broadcast_to_shape._always_inline_ = True + #ll_iter_broadcast_to_shape._annspecialcase_ = "specialize:argtype(1, 2)" + _gen_iter_funcs_cache[ndim] = ll_iter_new, ll_iter_broadcast_to_shape return ll_iter_new, ll_iter_broadcast_to_shape -def ll_unary_op(p0, p1, op=lambda x:x): - p0[0] = op(p1[0]) - -def ll_binary_op(p0, p1, p2, op=lambda x,y:x+y): - p0[0] = op(p1[0], p2[0]) - - -def ll_array_set(it0, it1): +def ll_array_set(ITEM, it0, it1): if it0.size == 0: return # empty LHS.. debug_assert(it0.size == it1.size, "it0.size == it1.size") while it0.index < it0.size: - it0.dataptr[0] = it1.dataptr[0] + it0.dataptr[0] = cast_primitive(ITEM, it1.dataptr[0]) it0.ll_next() it1.ll_next() @@ -244,7 +249,15 @@ self.itemsize = sizeof(self.ITEM) self.ARRAY, self.INDEXARRAY = self.make_types(self.ndim, self.ITEM) self.lowleveltype = self.ARRAY - self.ITER = ARRAY_ITER(self.ARRAY, self.INDEXARRAY) + self.ITER = ARRAY_ITER(self.ARRAY, self.INDEXARRAY, self.ITEM) + + def build_ITER_for(self, r_array): + # We need to iterate over r_array (which may have a + # different item type), but broadcast to self. + if self.ITER.TO.dataptr.TO == r_array.ITEM: + return self.ITER.TO + ITER = ARRAY_ITER(self.ARRAY, self.INDEXARRAY, r_array.ITEM) + return ITER.TO def build_from_array(self, hop, v_array): cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) @@ -328,11 +341,14 @@ # from the largest of the two args: v_array0 = hop.gendirectcall(ll_build_like, cARRAY, v_array1, v_array2) iter_new, iter_broadcast = gen_iter_funcs(r_array0.ndim) - cITER = hop.inputconst(Void, r_array0.ITER.TO) + iter_new._annenforceargs_ = [None, None, None, None] + cITER0 = hop.inputconst(Void, r_array0.ITER.TO) + cITER1 = hop.inputconst(Void, r_array0.build_ITER_for(r_array1)) + cITER2 = hop.inputconst(Void, r_array0.build_ITER_for(r_array2)) cbroadcast = hop.inputconst(Void, iter_broadcast) - v_it0 = hop.gendirectcall(iter_new, cITER, v_array0, v_array0, cbroadcast) - v_it1 = hop.gendirectcall(iter_new, cITER, v_array1, v_array0, cbroadcast) - v_it2 = hop.gendirectcall(iter_new, cITER, v_array2, v_array0, cbroadcast) + v_it0 = hop.gendirectcall(iter_new, cITER0, v_array0, v_array0, cbroadcast) + v_it1 = hop.gendirectcall(iter_new, cITER1, v_array1, v_array0, cbroadcast) + v_it2 = hop.gendirectcall(iter_new, cITER2, v_array2, v_array0, cbroadcast) cbinop = hop.inputconst(Void, binop) hop.gendirectcall(ll_array_binop, v_it0, v_it1, v_it2, cbinop) return v_array0 @@ -449,15 +465,21 @@ v_tuple = rtuple.newtuple(llops, r_tuple, [v_slc]) return r_tuple, v_tuple + def convert_list_to_array(r_list, v_list, llops): # [...] -> array([...]) from pypy.rpython.rmodel import inputconst ITEM = r_list.item_repr.lowleveltype - ARRAY, _ = ArrayRepr.make_types(1, ITEM) - cARRAY = inputconst(Void, ARRAY.TO) + s_item = lltype_to_annotation(ITEM) + key = type(s_item), s_item.knowntype + typecode = aarray.numpy_typedict.get(key, None) + ndim = 1 + s_array = SomeArray(typecode, ndim) + r_array = llops.rtyper.getrepr(s_array) + cARRAY = inputconst(Void, r_array.ARRAY.TO) v_array = llops.gendirectcall(ll_build_from_list, cARRAY, v_list) # XX does a copy :P #v_array = llops.gendirectcall(ll_build_alias_to_list, cARRAY, v_list) # nice idea... - return v_array + return r_array, v_array def convert_scalar_to_array(r_array, v_item, llops): # x -> array([x]) @@ -532,13 +554,15 @@ r_item, v_item = convert_scalar_to_array(r_view, v_item, hop.llops) source_ndim = 1 elif isinstance(r_item, AbstractBaseListRepr): - v_item = convert_list_to_array(r_item, v_item, hop.llops) + r_item, v_item = convert_list_to_array(r_item, v_item, hop.llops) source_ndim = 1 else: raise TypeError("bad item: %s"%r_item) assert source_ndim <= ndim + cITER = hop.inputconst(Void, r_view.build_ITER_for(r_item)) + cITEM = hop.inputconst(Void, r_array.ITEM) v_it1 = hop.gendirectcall(iter_new, cITER, v_item, v_view, cbroadcast) - return hop.gendirectcall(ll_array_set, v_it0, v_it1) + return hop.gendirectcall(ll_array_set, cITEM, v_it0, v_it1) class __extend__(pairtype(ArrayRepr, ArrayRepr)): def convert_from_to((r_array0, r_array1), v, llops): @@ -640,24 +664,6 @@ i -= 1 return array -#def ll_setitem1(array, index, item): -# array.data[index] = item -# -#def ll_getitem1(array, index): -# return array.data[index] - -#def ll_add(ARRAY, a1, a2): -# size = a1.shape[0] -# if size != a2.shape[0]: -# raise ValueError -# array = malloc(ARRAY) -# array.data = malloc(ARRAY.data.TO, size) -# i = 0 -# while i < size: -# array.data[i] = a1.data[i] + a2.data[i] -# i += 1 -# array.dataptr = direct_arrayitems(array.data) -# return array def ll_transpose(ARRAY, ao): ndim = ao.ndim Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Tue Sep 11 21:30:48 2007 @@ -367,7 +367,6 @@ assert res.dataptr[i] == data[i] def test_specialize_list_coerce(self): - py.test.skip('coercion not implemented') def f(): a = numpy.zeros((3,4), dtype='i') a[:] = [3.,4.,4.,2.] @@ -378,7 +377,6 @@ assert res.dataptr[i] == data[i] def test_specialize_rhs_coerce(self): - py.test.skip('coercion not implemented') def f(): a = numpy.zeros((4,), dtype='i') b = numpy.array([3.,4.,4.,2.]) @@ -580,7 +578,6 @@ assert res.data[1] == 11 def test_specialize_array_add_1_1_coerce(self): - py.test.skip('coercion not implemented') def f(): a1 = numpy.array([1,2]) a2 = numpy.array([6.,9.]) @@ -645,7 +642,9 @@ return a[ii, jj] fn = self.compile(f, [int, int]) - assert fn(2,3) == 2*3 + for i in range(4): + for j in range(5): + assert fn(i, j) == i*j def test_compile_view(self): def f(ii, jj): From arigo at codespeak.net Tue Sep 11 21:56:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Sep 2007 21:56:11 +0200 (CEST) Subject: [pypy-svn] r46485 - pypy/branch/cleanup-weakref/pypy/module/_weakref Message-ID: <20070911195611.D2E8080F0@code0.codespeak.net> Author: arigo Date: Tue Sep 11 21:56:11 2007 New Revision: 46485 Modified: pypy/branch/cleanup-weakref/pypy/module/_weakref/__init__.py Log: Fix the logic used to skip the _weakref app-level module from the list of enabled modules. Modified: pypy/branch/cleanup-weakref/pypy/module/_weakref/__init__.py ============================================================================== --- pypy/branch/cleanup-weakref/pypy/module/_weakref/__init__.py (original) +++ pypy/branch/cleanup-weakref/pypy/module/_weakref/__init__.py Tue Sep 11 21:56:11 2007 @@ -14,7 +14,6 @@ } def __init__(self, space, *args): - if space.config.translation.sandbox: - raise SkipModule( - "XXX weakrefs are disabled in a sandbox translation ATM") + if not space.config.translation.rweakref: + raise SkipModule("no RPython-level weakrefs") MixedModule.__init__(self, space, *args) From antocuni at codespeak.net Tue Sep 11 21:57:02 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 11 Sep 2007 21:57:02 +0200 (CEST) Subject: [pypy-svn] r46486 - in pypy/dist/pypy/translator/jvm: src/pypy test Message-ID: <20070911195702.AA8D78092@code0.codespeak.net> Author: antocuni Date: Tue Sep 11 21:57:02 2007 New Revision: 46486 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java pypy/dist/pypy/translator/jvm/test/test_builtin.py Log: os.read works with genjvm! Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Tue Sep 11 21:57:02 2007 @@ -60,6 +60,49 @@ } } +class RandomAccessFileWrapper extends FileWrapper +{ + private RandomAccessFile file; + private boolean canRead; + private boolean canWrite; + + public RandomAccessFileWrapper(RandomAccessFile file, boolean canRead, boolean canWrite) + { + this.file = file; + this.canRead = canRead; + this.canWrite = canWrite; + } + + public void write(String buffer) + { + if (!this.canWrite) + ll_os.throwOSError(PyPy.EBADF, "Cannot write to this fd"); + + try { + this.file.writeChars(buffer); + } + catch(IOException e) { + ll_os.throwOSError(PyPy.EIO, e.getMessage()); + } + } + + public String read(int count) + { + if (!this.canRead) + ll_os.throwOSError(PyPy.EBADF, "Cannot read from this fd"); + + try { + byte[] buffer = new byte[count]; + int n = this.file.read(buffer); + return new String(buffer, 0, n); + } + catch(IOException e) { + ll_os.throwOSError(PyPy.EIO, e.getMessage()); + return null; // never reached + } + } +} + public class ll_os { @@ -80,6 +123,7 @@ private static final int S_IFDIR = 16384; private static final int S_IFREG = 32768; + private static int fdcount; private static Map FileDescriptors = new HashMap(); private static Map ErrorMessages = new HashMap(); @@ -87,6 +131,7 @@ FileDescriptors.put(new Integer(0), new PrintStreamWrapper(System.out)); FileDescriptors.put(new Integer(1), new InputStreamWrapper(System.in)); FileDescriptors.put(new Integer(2), new PrintStreamWrapper(System.err)); + fdcount = 2; } public static void throwOSError(int errno, String errText) { @@ -94,10 +139,74 @@ PyPy.interlink.throwOSError(errno); } + private static FileWrapper getfd(int fd) + { + FileWrapper f = FileDescriptors.get(new Integer(fd)); + if (f == null) + throwOSError(PyPy.EBADF, "Invalid file descriptor: " + fd); + return f; + } + + private static RandomAccessFile open_file(String name, String javaMode, int flags) + { + RandomAccessFile file; + + try { + file = new RandomAccessFile(name, javaMode); + } + catch(IOException e) { + throwOSError(PyPy.ENOENT, e.getMessage()); + return null; + } + + try { + if ((flags & O_TRUNC) !=0 ) + file.setLength(0); + + if ((flags & O_APPEND) !=0 ) + file.seek(file.length()-1); + } + catch(IOException e) { + throwOSError(PyPy.EPERM, e.getMessage()); + return null; + } + + return file; + } + public static int ll_os_open(String name, int flags, int mode) { - throwOSError(PyPy.ENOENT, "DUMMY ll_os_open"); - return -1; + boolean canRead = false; + boolean canWrite = false; + + if ((flags & O_RDWR) != 0) { + canRead = true; + canWrite = true; + } + else if ((flags & O_WRONLY) != 0) + canWrite = true; + else + canRead = true; + + String javaMode = canWrite ? "rw" : "r"; + + // XXX: we ignore O_CREAT + RandomAccessFile file = open_file(name, javaMode, flags); + RandomAccessFileWrapper wrapper = new RandomAccessFileWrapper(file, canRead, canWrite); + + fdcount++; + FileDescriptors.put(new Integer(fdcount), wrapper); + return fdcount; + } + + public static String ll_os_read(int fd, int count) + { + return getfd(fd).read(count); + } + + public static String ll_os_read(int fd, long count) + { + return ll_os_read(fd, (int)count); } public static StatResult ll_os_lstat(String path) Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Tue Sep 11 21:57:02 2007 @@ -11,9 +11,6 @@ def test_os_write_magic(self): py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - def test_os_read(self): - py.test.skip("ll_os_open is not currently implemented in the Jvm backed") - def test_builtin_math_frexp(self): py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") From antocuni at codespeak.net Tue Sep 11 22:14:57 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 11 Sep 2007 22:14:57 +0200 (CEST) Subject: [pypy-svn] r46487 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070911201457.0569880EF@code0.codespeak.net> Author: antocuni Date: Tue Sep 11 22:14:57 2007 New Revision: 46487 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Log: implement ll_os_close; fix a bug in ll_os_open when the EOF is reached. Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Tue Sep 11 22:14:57 2007 @@ -9,6 +9,7 @@ { public abstract void write(String buffer); public abstract String read(int count); + public abstract void close(); } class PrintStreamWrapper extends FileWrapper @@ -30,6 +31,11 @@ ll_os.throwOSError(PyPy.EBADF, "Write-only fd"); return null; // never reached } + + public void close() + { + ll_os.throwOSError(PyPy.EBADF, "Cannot close stdout or stderr"); + } } class InputStreamWrapper extends FileWrapper @@ -58,6 +64,11 @@ return null; // never reached } } + + public void close() + { + ll_os.throwOSError(PyPy.EBADF, "Cannot close stdin"); + } } class RandomAccessFileWrapper extends FileWrapper @@ -94,13 +105,26 @@ try { byte[] buffer = new byte[count]; int n = this.file.read(buffer); - return new String(buffer, 0, n); + if (n == -1) + return ""; // XXX: is it right? + else + return new String(buffer, 0, n); } catch(IOException e) { ll_os.throwOSError(PyPy.EIO, e.getMessage()); return null; // never reached } } + + public void close() + { + try { + this.file.close(); + } + catch(IOException e) { + ll_os.throwOSError(PyPy.EIO, e.getMessage()); + } + } } @@ -199,6 +223,13 @@ return fdcount; } + public static void ll_os_close(int fd) + { + FileWrapper wrapper = getfd(fd); + wrapper.close(); + FileDescriptors.remove(new Integer(fd)); + } + public static String ll_os_read(int fd, int count) { return getfd(fd).read(count); From antocuni at codespeak.net Wed Sep 12 00:13:07 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 00:13:07 +0200 (CEST) Subject: [pypy-svn] r46488 - in pypy/dist/pypy/translator/jvm: . test Message-ID: <20070911221307.2772280A4@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 00:13:05 2007 New Revision: 46488 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/test/test_list.py Log: we can't assume MALLOC_ZERO_FILLED for jvm! Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Wed Sep 12 00:13:05 2007 @@ -1077,7 +1077,10 @@ def push_null(self, OOTYPE): self.emit(ACONST_NULL) - DEFINED_INT_SYMBOLICS = {'MALLOC_ZERO_FILLED':1, + # we can't assume MALLOC_ZERO_FILLED, because for scalar type the + # default item for ArrayList is null, not e.g. Integer(0) or + # Char(0). + DEFINED_INT_SYMBOLICS = {'MALLOC_ZERO_FILLED':0, '0 /* we are not jitted here */': 0} def push_primitive_constant(self, TYPE, value): Modified: pypy/dist/pypy/translator/jvm/test/test_list.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_list.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_list.py Wed Sep 12 00:13:05 2007 @@ -8,3 +8,11 @@ def test_getitem_exc(self): py.test.skip('fixme!') + + def test_zeroed_list(self): + def fn(): + lst = [0] * 16 + return lst[0] + res = self.interpret(fn, []) + assert res == 0 + From simonb at codespeak.net Wed Sep 12 05:03:24 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Wed, 12 Sep 2007 05:03:24 +0200 (CEST) Subject: [pypy-svn] r46489 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070912030324.69D0E80D5@code0.codespeak.net> Author: simonb Date: Wed Sep 12 05:03:23 2007 New Revision: 46489 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: annotating and rtypeing of astype method Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Wed Sep 12 05:03:23 2007 @@ -82,6 +82,12 @@ def method_transpose(self): return SomeArray(self.typecode, self.ndim) + def method_astype(self, s_dtype): + if isinstance(s_dtype, SomeChar) and s_dtype.is_constant(): + typecode = s_dtype.const + return SomeArray(typecode, self.ndim) + raise AnnotatorError() + class __extend__(pairtype(SomeArray, SomeArray)): def add((s_arr1, s_arr2)): Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Wed Sep 12 05:03:23 2007 @@ -295,6 +295,23 @@ cARRAY = inputconst(lltype.Void, r_result.lowleveltype.TO) return hop.llops.gendirectcall(ll_build_alias_shape, cARRAY, v_array, v_tuple) + def rtype_method_astype(r_array, hop): + r_result = hop.r_result + v_array = hop.inputarg(r_array, 0) + cARRAY = inputconst(lltype.Void, r_result.lowleveltype.TO) + v_result = hop.llops.gendirectcall(ll_build_like, cARRAY, v_array) + + iter_new, iter_broadcast = gen_iter_funcs(r_array.ndim) + cbroadcast = hop.inputconst(Void, iter_broadcast) + cITER0 = hop.inputconst(Void, r_result.ITER.TO) + v_it0 = hop.gendirectcall(iter_new, cITER0, v_result, v_result, cbroadcast) + cITER1 = hop.inputconst(Void, r_array.ITER.TO) + v_it1 = hop.gendirectcall(iter_new, cITER1, v_array, v_array, cbroadcast) + cITEM = hop.inputconst(Void, r_result.ITEM) + hop.gendirectcall(ll_array_set, cITEM, v_it0, v_it1) + return v_result + + def get_ndim(self, hop, v_array): cname = inputconst(Void, 'ndim') return hop.llops.genop('getfield', [v_array, cname], resulttype=Signed) @@ -339,7 +356,7 @@ cARRAY = hop.inputconst(Void, r_array0.ARRAY.TO) # We build a contiguous "result" array # from the largest of the two args: - v_array0 = hop.gendirectcall(ll_build_like, cARRAY, v_array1, v_array2) + v_array0 = hop.gendirectcall(ll_build_like2, cARRAY, v_array1, v_array2) iter_new, iter_broadcast = gen_iter_funcs(r_array0.ndim) iter_new._annenforceargs_ = [None, None, None, None] cITER0 = hop.inputconst(Void, r_array0.ITER.TO) @@ -641,7 +658,23 @@ array.dataptr = ao.dataptr return array -def ll_build_like(ARRAY, array0, array1): +def ll_build_like(ARRAY, array0): + ndim = array0.ndim + array = ARRAY.ll_allocate(ndim) + sz = ll_mul_list(array0.shape, array0.ndim) + array.data = malloc(ARRAY.data.TO, sz) + array.dataptr = direct_arrayitems(array.data) + itemsize = 1 + i = ndim - 1 + while i >= 0: + size = array0.shape[i] + array.shape[i] = size + array.strides[i] = itemsize + itemsize *= size + i -= 1 + return array + +def ll_build_like2(ARRAY, array0, array1): # Build with shape from the largest of array0 or array1. # Note we cannot take the union of array0 and array1. ndim = max(array0.ndim, array1.ndim) Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Wed Sep 12 05:03:23 2007 @@ -188,6 +188,15 @@ assert type(s) == SomeArray assert s.ndim == 2 + def f_astype(): + a = numpy.array(range(12)) + return a.astype('d') + + s = a.build_types(f_astype, []) + assert type(s) == SomeArray + assert s.ndim == 1 + assert s.typecode == 'd' + def test_annotate_indexing(self): def f(): a = numpy.empty((4,3), dtype='i') @@ -330,6 +339,16 @@ assert res.dataptr[5] == 5 assert res.dataptr[6] == 0 + def f_astype(): + a = numpy.array(range(12)) + b = a.astype('d') + b = b/2 + return b + + res = interpret(f_astype, []) + assert res.dataptr[0] == 0. + assert res.dataptr[1] == 0.5 + def test_specialize_view_0(self): def f(): a = numpy.empty((4,3), dtype='i') @@ -657,3 +676,4 @@ assert fn(2,3) == 0 + From antocuni at codespeak.net Wed Sep 12 11:05:52 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 11:05:52 +0200 (CEST) Subject: [pypy-svn] r46490 - pypy/dist/pypy/translator/oosupport/test_template Message-ID: <20070912090552.E3AEE80F0@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 11:05:51 2007 New Revision: 46490 Modified: pypy/dist/pypy/translator/oosupport/test_template/runtest.py Log: fix this test Modified: pypy/dist/pypy/translator/oosupport/test_template/runtest.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/runtest.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/runtest.py Wed Sep 12 11:05:51 2007 @@ -10,11 +10,11 @@ def test_patch_os(self): import os - from pypy.translator.cli.support import patch, unpatch, NT_OS + from pypy.translator.oosupport.support import patch_os, unpatch_os, NT_OS original_O_CREAT = os.O_CREAT - olddefs = patch() + olddefs = patch_os() assert os.O_CREAT == NT_OS['O_CREAT'] - unpatch(*olddefs) + unpatch_os(olddefs) assert os.O_CREAT == original_O_CREAT def test_int(self): From antocuni at codespeak.net Wed Sep 12 11:31:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 11:31:00 +0200 (CEST) Subject: [pypy-svn] r46491 - pypy/dist/pypy/rpython/test Message-ID: <20070912093100.81F8380F7@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 11:31:00 2007 New Revision: 46491 Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py Log: a test to os.lseek. It looks like it is broken on lltype :-( Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Wed Sep 12 11:31:00 2007 @@ -215,6 +215,33 @@ res = self.interpret(f, []) assert self.ll_to_string(res) == 'hello world' + def test_os_lseek(self): + self._skip_llinterpreter("os.lseek", skipOO=False) + import os + tmpfile = str(udir.udir.join("os_lseek_test")) + f = file(tmpfile, 'w') + f.write('0123456789') + f.close() + SEEK_SET = 0 + SEEK_CUR = 1 + SEEK_END = 2 + def fn(): + fd = os.open(tmpfile, os.O_RDONLY, 0777) + res = '' + os.lseek(fd, 5, SEEK_SET) + res += os.read(fd, 1) + os.lseek(fd, 2, SEEK_CUR) + res += os.read(fd, 1) + os.lseek(fd, -2, SEEK_CUR) + res += os.read(fd, 1) + os.lseek(fd, -1, SEEK_END) + res += os.read(fd, 1) + os.close(fd) + return res + res1 = fn() + res2 = self.ll_to_string(self.interpret(fn, [])) + assert res1 == res2 + def test_os_dup(self): import os def fn(fd): From rxe at codespeak.net Wed Sep 12 12:05:50 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 12 Sep 2007 12:05:50 +0200 (CEST) Subject: [pypy-svn] r46492 - pypy/dist/pypy/translator/llvm/test Message-ID: <20070912100550.7B67280F8@code0.codespeak.net> Author: rxe Date: Wed Sep 12 12:05:49 2007 New Revision: 46492 Added: pypy/dist/pypy/translator/llvm/test/test_rffi.py pypy/dist/pypy/translator/llvm/test/test_threading.py Modified: pypy/dist/pypy/translator/llvm/test/runtest.py pypy/dist/pypy/translator/llvm/test/test_extfunc.py pypy/dist/pypy/translator/llvm/test/test_genllvm1.py pypy/dist/pypy/translator/llvm/test/test_lltype.py pypy/dist/pypy/translator/llvm/test/test_rtagged.py Log: just shuffling tests around. they are all still skipped Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Wed Sep 12 12:05:49 2007 @@ -1,6 +1,6 @@ import py -py.test.skip("llvm is no longer actively maintained") -from pypy.tool import isolate +py.test.skip("llvm is not actively maintained") + from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM @@ -9,14 +9,16 @@ ext_modules = [] -# test options +# prevents resource leaking +use_isolate = True + +# if test can't be run using isolate, skip the test (useful for buildbots) run_isolated_only = True -do_not_isolate = False from pypy import conftest def _cleanup(leave=0): - # no test should ever need more than 5 compiled functions + from pypy.tool import isolate if leave: mods = ext_modules[:-leave] else: @@ -40,13 +42,6 @@ py.test.skip("llvm version not up-to-date (found " "%.1f, should be >= %.1f)" % (llvm_ver, MINIMUM_LLVM_VERSION)) -def gcc3_test(): - gcc_ver = gcc_version() - if int(gcc_ver) != 3: - py.test.skip("test required gcc version 3 (found version %.1f)" % gcc_ver) - return False - return True - #______________________________________________________________________________ def genllvm_compile(function, @@ -89,14 +84,15 @@ driver.translator.view() return driver.c_module, driver.c_entryp -def compile_test(function, annotation, isolate=True, **kwds): +def compile_test(function, annotation, isolate_hint=True, **kwds): " returns module and compiled function " llvm_test() - if run_isolated_only and not isolate: - py.test.skip("skipping not isolated test") + + if run_isolated_only and not isolate_hint: + py.test.skip("skipping unrecommended isolated test") # turn off isolation? - isolate = isolate and not do_not_isolate + isolate = use_isolate and isolate_hint # maintain only 3 isolated process (if any) _cleanup(leave=3) @@ -107,7 +103,7 @@ ext_modules.append(mod) return mod, fn -def compile_function(function, annotation, isolate=True, **kwds): +def compile_function(function, annotation, isolate_hint=True, **kwds): " returns compiled function " - return compile_test(function, annotation, isolate=isolate, **kwds)[1] + return compile_test(function, annotation, isolate_hint=isolate_hint, **kwds)[1] Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_extfunc.py Wed Sep 12 12:05:49 2007 @@ -4,21 +4,9 @@ import sys import py -from pypy.tool.udir import udir -from pypy.rlib.rarithmetic import r_uint - -py.test.skip("Extfunc support in llvm needs refactoring") -# XXX in particular, try to share the tests from c/test/test_extfunc! from pypy.translator.llvm.test.runtest import * -def test_external_function_ll_os_dup(): - def fn(): - return os.dup(0) - f = compile_function(fn, [], isolate=False) - fn() - assert os.path.sameopenfile(f(), fn()) - def test_external_function_ll_time_time(): import time def fn(): @@ -30,135 +18,15 @@ import time def fn(): return time.clock() - f = compile_function(fn, [], isolate=False) + f = compile_function(fn, [], isolate_hint=False) assert abs(f()-fn()) < 10.0 -def test_external_function_ll_time_sleep(): - import time - def fn(t): - time.sleep(t) - return 666 - f = compile_function(fn, [float]) - start_time = time.time() - delay_time = 2.0 - d = f(delay_time) - duration = time.time() - start_time - assert duration >= delay_time - 0.5 - assert duration <= delay_time + 0.5 - -path = str(udir.join("e")) - -def test_os_file_ops_open_close(): - def openclose(): - fd = os.open(path, os.O_CREAT|os.O_RDWR, 0777) - os.close(fd) - return fd - - if os.path.exists(path): - os.unlink(path) - f = compile_function(openclose, []) - result = f() - assert os.path.exists(path) - -def test_os_file_ops_open_write_close(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") - def openwriteclose(): - fd = os.open(path, os.O_CREAT|os.O_RDWR, 0777) - byteswritten = os.write(fd, path) - os.close(fd) - return byteswritten - - if os.path.exists(path): - os.unlink(path) - f = compile_function(openwriteclose, []) - result = f() - assert os.path.exists(path) - assert open(path).read() == path - -def test_os_file_ops_open_write_read_close(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") - def openwriteclose_openreadclose(): - fd = os.open(path, os.O_CREAT|os.O_RDWR, 0777) - byteswritten = os.write(fd, path+path+path) - os.close(fd) - - fd = os.open(path, os.O_RDWR, 0777) - maxread = 1000 - r = os.read(fd, maxread) - os.close(fd) - - return len(r) - - if os.path.exists(path): - os.unlink(path) - f = compile_function(openwriteclose_openreadclose, []) - result = f() - assert os.path.exists(path) - assert open(path).read() == path * 3 - assert result is len(path) * 3 - -# following from translator/c/test/test_extfunc.py Revision: 15320 (jul 29th 2005) - -def test_os_stat(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") - filename = str(py.magic.autopath()) - def call_stat0(): - st = os.stat(filename) - return st[0] - def call_stat1(): - st = os.stat(filename) - return st[1] - def call_stat2(): - st = os.stat(filename) - return st[2] - f0 = compile_function(call_stat0, []) - f1 = compile_function(call_stat1, []) - f2 = compile_function(call_stat2, []) - st = os.stat(filename) - assert f0() == st[0] - assert f1() == st[1] - assert f2() == st[2] +def test_math_frexp(): + py.test.skip("what the hell is going on!") -def test_os_fstat(): if sys.maxint != 2**31-1: py.test.skip("WIP on 64 bit architectures") - filename = str(py.magic.autopath()) - def call_fstat0(): - fd = os.open(filename, os.O_RDONLY, 0777) - st = os.fstat(fd) - os.close(fd) - return st[0] - def call_fstat1(): - fd = os.open(filename, os.O_RDONLY, 0777) - st = os.fstat(fd) - os.close(fd) - return st[1] - def call_fstat2(): - fd = os.open(filename, os.O_RDONLY, 0777) - st = os.fstat(fd) - os.close(fd) - return st[2] - f0 = compile_function(call_fstat0, []) - f1 = compile_function(call_fstat1, []) - f2 = compile_function(call_fstat2, []) - st = os.stat(filename) - assert f0() == st[0] - assert f1() == st[1] - assert f2() == st[2] - -def test_os_getcwd(): - cwd = os.getcwd() - def does_stuff(): - return os.getcwd() == cwd - f1 = compile_function(does_stuff, []) - assert f1() -def test_math_frexp(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") from math import frexp def fn(x): res = frexp(x) @@ -168,6 +36,8 @@ assert res == fn(10.123) def test_math_modf(): + py.test.skip("what the hell is going on!") + from math import modf def fn(x): res = modf(x) @@ -196,54 +66,6 @@ for funcname in simple_math_functions: yield math_function_test, funcname -def test_os_path_exists(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") - tmpfile = str(udir.join('test_os_path_exists.TMP')) - def fn(): - return os.path.exists(tmpfile) - f = compile_function(fn, []) - open(tmpfile, 'w').close() - assert f() == True - os.unlink(tmpfile) - assert f() == False - -def test_dynamic_string_null_termination(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") - # forces malloc / versus pbc for NUL testing of C string - tmpfile = str(udir.join('test_os_path_exists.TMP')) - def fn(l): - filename = tmpfile[:l] - return os.path.exists(filename) - f = compile_function(fn, [r_uint]) - open(tmpfile, 'w').close() - lfile = len(tmpfile) - assert f(lfile) == True - assert f(lfile-2) == False - -def test_os_path_isdir(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") - directory = "./." - def fn(): - return os.path.isdir(directory) - f = compile_function(fn, []) - assert f() == True - directory = "some/random/name" - def fn(): - return os.path.isdir(directory) - f = compile_function(fn, []) - assert f() == False - -def test_os_isatty(): - def call_isatty(fd): - return os.isatty(fd) - f = compile_function(call_isatty, [int], isolate=False) - assert f(0) == os.isatty(0) - assert f(1) == os.isatty(1) - assert f(2) == os.isatty(2) - def test_rarith_parts_to_float(): if sys.maxint != 2**31-1: py.test.skip("WIP on 64 bit architectures") @@ -276,305 +98,3 @@ for i, s in enumerate(as_string): assert f(i) -def test_os_unlink(): - tmpfile = str(udir.join('test_os_path_exists.TMP')) - def fn(): - os.unlink(tmpfile) - return 0 - f = compile_function(fn, []) - open(tmpfile, 'w').close() - fn() - assert not os.path.exists(tmpfile) - -def test_chdir(): - path = '..' - def does_stuff(): - os.chdir(path) - return 0 - f1 = compile_function(does_stuff, []) - curdir = os.getcwd() - try: - os.chdir() - except: pass # toplevel - def does_stuff2(): - os.chdir(curdir) - return 0 - f1 = compile_function(does_stuff2, []) - f1() - assert curdir == os.getcwd() - -def test_mkdir_rmdir(): - path = str(udir.join('test_mkdir_rmdir')) - def does_stuff(delete): - if delete: - os.rmdir(path) - else: - os.mkdir(path, 0777) - return 0 - f1 = compile_function(does_stuff, [bool]) - f1(False) - assert os.path.exists(path) and os.path.isdir(path) - f1(True) - assert not os.path.exists(path) - -# more from translator/c/test/test_extfunc.py Revision: 19054 - - -def test_lock(): - py.test.skip("XXX does not work with exception transform (why not?)") - import thread - import pypy.module.thread.rpython.exttable # for declare()/declaretype() - def fn(): - l = thread.allocate_lock() - ok1 = l.acquire(True) - ok2 = l.acquire(False) - l.release() - ok2_and_a_half = False - try: - l.release() - except thread.error: - ok2_and_a_half = True - ok3 = l.acquire(False) - return ok1 and not ok2 and ok2_and_a_half and ok3 - f = compile_function(fn, []) - res = f() - assert res - -def test_simple_start_new_thread(): - import thread - import pypy.module.thread.rpython.exttable # for declare()/declaretype() - class Arg: - pass - def mythreadedfunction(arg): - assert arg.value == 42 - def myotherthreadedfunction(arg): - assert arg.value == 43 - a42 = Arg() - a42.value = 42 - a43 = Arg() - a43.value = 43 - def fn(i): - thread.start_new_thread(mythreadedfunction, (a42,)) - thread.start_new_thread(myotherthreadedfunction, (a43,)) - if i == 1: - x = mythreadedfunction - a = a42 - else: - x = myotherthreadedfunction - a = a43 - thread.start_new_thread(x, (a,)) - return 42 - f = compile_function(fn, [int]) - res = f(1) - assert res == 42 - -def test_start_new_thread(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") - import thread - import pypy.module.thread.rpython.exttable # for declare()/declaretype() - class Arg: - pass - a = Arg() - a.x = 5 - def mythreadedfunction(arg): - arg.x += 37 - arg.myident = thread.get_ident() - arg.lock.release() - def fn(): - a.lock = thread.allocate_lock() - a.lock.acquire(True) - ident = thread.start_new_thread(mythreadedfunction, (a,)) - assert ident != thread.get_ident() - a.lock.acquire(True) # wait for the thread to finish - assert a.myident == ident - return a.x - f = compile_function(fn, []) - res = f() - assert res == 42 - -def test_prebuilt_lock(): - import thread - import pypy.module.thread.rpython.exttable # for declare()/declaretype() - lock0 = thread.allocate_lock() - lock1 = thread.allocate_lock() - lock1.acquire() - def fn(i): - lock = [lock0, lock1][i] - ok = lock.acquire(False) - if ok: lock.release() - return ok - f = compile_function(fn, [int]) - res = f(0) - assert res == True - res = f(1) - assert res == False - -def test_pipe_dup_dup2(): - def does_stuff(): - a, b = os.pipe() - c = os.dup(a) - d = os.dup(b) - assert a != b - assert a != c - assert a != d - assert b != c - assert b != d - assert c != d - os.close(c) - os.dup2(d, c) - e, f = os.pipe() - assert e != a - assert e != b - assert e != c - assert e != d - assert f != a - assert f != b - assert f != c - assert f != d - assert f != e - os.close(a) - os.close(b) - os.close(c) - os.close(d) - os.close(e) - os.close(f) - return 42 - f1 = compile_function(does_stuff, []) - res = f1() - assert res == 42 - -def test_os_chmod(): - tmpfile = str(udir.join('test_os_chmod.txt')) - f = open(tmpfile, 'w') - f.close() - def does_stuff(mode): - os.chmod(tmpfile, mode) - return 0 - f1 = compile_function(does_stuff, [int]) - f1(0000) - assert os.stat(tmpfile).st_mode & 0777 == 0000 - f1(0644) - assert os.stat(tmpfile).st_mode & 0777 == 0644 - -def test_os_rename(): - tmpfile1 = str(udir.join('test_os_rename_1.txt')) - tmpfile2 = str(udir.join('test_os_rename_2.txt')) - f = open(tmpfile1, 'w') - f.close() - def does_stuff(): - os.rename(tmpfile1, tmpfile2) - return 0 - f1 = compile_function(does_stuff, []) - f1() - assert os.path.exists(tmpfile2) - assert not os.path.exists(tmpfile1) - -if hasattr(os, 'getpid'): - def test_os_getpid(): - def does_stuff(): - return os.getpid() - f1 = compile_function(does_stuff, [], isolate=False) - res = f1() - assert res == os.getpid() - -if hasattr(os, 'link'): - def test_links(): - import stat - tmpfile1 = str(udir.join('test_links_1.txt')) - tmpfile2 = str(udir.join('test_links_2.txt')) - tmpfile3 = str(udir.join('test_links_3.txt')) - f = open(tmpfile1, 'w') - f.close() - def does_stuff(): - os.symlink(tmpfile1, tmpfile2) - os.link(tmpfile1, tmpfile3) - assert os.readlink(tmpfile2) == tmpfile1 - flag= 0 - st = os.lstat(tmpfile1) - flag = flag*10 + stat.S_ISREG(st[0]) - flag = flag*10 + stat.S_ISLNK(st[0]) - st = os.lstat(tmpfile2) - flag = flag*10 + stat.S_ISREG(st[0]) - flag = flag*10 + stat.S_ISLNK(st[0]) - st = os.lstat(tmpfile3) - flag = flag*10 + stat.S_ISREG(st[0]) - flag = flag*10 + stat.S_ISLNK(st[0]) - return flag - f1 = compile_function(does_stuff, []) - res = f1() - assert res == 100110 - assert os.path.islink(tmpfile2) - assert not os.path.islink(tmpfile3) -if hasattr(os, 'fork'): - def test_fork(): - def does_stuff(): - pid = os.fork() - if pid == 0: # child - os._exit(4) - pid1, status1 = os.waitpid(pid, 0) - assert pid1 == pid - return status1 - f1 = compile_function(does_stuff, []) - status1 = f1() - assert os.WIFEXITED(status1) - assert os.WEXITSTATUS(status1) == 4 - -if hasattr(posix, 'execv'): - def test_execv(): - py.test.skip("not working yet") - filename = str(udir.join('test_execv.txt')) - executable = sys.executable - def does_stuff(): - progname = str(executable) - l = ['', ''] - l[0] = progname - l[1] = "-c" - l.append('open("%s","w").write("1")' % filename) - pid = os.fork() - if pid == 0: - os.execv(progname, l) - else: - os.waitpid(pid, 0) - return 1 - func = compile_function(does_stuff, []) - func() - assert open(filename).read() == "1" - - def test_execv_raising(): - py.test.skip("not working yet") - def does_stuff(): - l = [] - l.append("asddsadw32eewdfwqdqwdqwd") - try: - os.execv(l[0], l) - except OSError: - return 1 - else: - return 0 - func = compile_function(does_stuff, []) - res = func() - assert res == 1 - - def test_execve(): - py.test.skip("not working yet") - filename = str(udir.join('test_execve.txt')) - executable = sys.executable - def does_stuff(): - progname = executable - l = [] - l.append(progname) - l.append("-c") - l.append('import os; open("%s", "w").write(os.environ["STH"])' % filename) - env = {} - env["STH"] = "42" - env["sthelse"] = "a" - pid = os.fork() - if pid == 0: - os.execve(progname, l, env) - else: - os.waitpid(pid, 0) - return 1 - func = compile_function(does_stuff, []) - func() - assert open(filename).read() == "42" Modified: pypy/dist/pypy/translator/llvm/test/test_genllvm1.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_genllvm1.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_genllvm1.py Wed Sep 12 12:05:49 2007 @@ -70,7 +70,7 @@ class TestString(object): def test_f2(self): f = compile_function(llvmsnippet.string_f2, [int, int]) - assert chr(f(1, 0)) == "a" + assert f(1, 0) == "a" class TestPBC(object): Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Wed Sep 12 12:05:49 2007 @@ -240,7 +240,6 @@ assert f() == floats_fn() def test_fixedsizearray(): - gcc3_test() S = Struct("s", ('v', Signed)) A7 = FixedSizeArray(Signed, 7) A3 = FixedSizeArray(S, 3) @@ -269,7 +268,6 @@ assert fn() == 607 def test_recursivearray(): - gcc3_test() A = ForwardReference() A.become(FixedSizeArray(Struct("S", ('a', Ptr(A))), 5)) TREE = GcStruct("TREE", ("root", A), ("other", A)) @@ -300,7 +298,6 @@ fn() def test_call_with_fixedsizearray(): - gcc3_test() A = FixedSizeArray(Struct('s1', ('x', Signed)), 5) S = GcStruct('s', ('a', Ptr(A))) a = malloc(A, immortal=True) @@ -316,7 +313,6 @@ assert res == 123 def test_more_prebuilt_arrays(): - gcc3_test() A = FixedSizeArray(Struct('s1', ('x', Signed)), 5) S = GcStruct('s', ('a1', Ptr(A)), ('a2', A)) s = malloc(S, zero=True) @@ -336,7 +332,6 @@ assert res == 60 def test_fnptr_with_fixedsizearray(): - gcc3_test() A = ForwardReference() F = FuncType([Ptr(A)], Signed) A.become(FixedSizeArray(Struct('s1', ('f', Ptr(F)), ('n', Signed)), 5)) @@ -385,7 +380,6 @@ assert res == 0 + 10 + 30 + 1000 def test_direct_fieldptr(): - gcc3_test() S = GcStruct('S', ('x', Signed), ('y', Signed)) def llf(n): s = malloc(S) Added: pypy/dist/pypy/translator/llvm/test/test_rffi.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rffi.py Wed Sep 12 12:05:49 2007 @@ -0,0 +1,401 @@ +import os +import sys + +import py +from pypy.tool.udir import udir +from pypy.rlib.rarithmetic import r_uint + +py.test.skip("Extfunc support in llvm needs refactoring (!!??!)") + +from pypy.translator.llvm.test.runtest import * + +def test_external_function_ll_os_dup(): + def fn(): + return os.dup(0) + f = compile_function(fn, [], isolate_hint=False) + fn() + assert os.path.sameopenfile(f(), fn()) + +def test_external_function_ll_time_sleep(): + import time + def fn(t): + time.sleep(t) + return 666 + f = compile_function(fn, [float]) + start_time = time.time() + delay_time = 2.0 + d = f(delay_time) + duration = time.time() - start_time + assert duration >= delay_time - 0.5 + assert duration <= delay_time + 0.5 + +path = str(udir.join("e")) + +def test_os_file_ops_open_close(): + def openclose(): + fd = os.open(path, os.O_CREAT|os.O_RDWR, 0777) + os.close(fd) + return fd + + if os.path.exists(path): + os.unlink(path) + f = compile_function(openclose, []) + result = f() + assert os.path.exists(path) + +def test_os_file_ops_open_write_close(): + if sys.maxint != 2**31-1: + py.test.skip("WIP on 64 bit architectures") + def openwriteclose(): + fd = os.open(path, os.O_CREAT|os.O_RDWR, 0777) + byteswritten = os.write(fd, path) + os.close(fd) + return byteswritten + + if os.path.exists(path): + os.unlink(path) + f = compile_function(openwriteclose, []) + result = f() + assert os.path.exists(path) + assert open(path).read() == path + +def test_os_file_ops_open_write_read_close(): + if sys.maxint != 2**31-1: + py.test.skip("WIP on 64 bit architectures") + def openwriteclose_openreadclose(): + fd = os.open(path, os.O_CREAT|os.O_RDWR, 0777) + byteswritten = os.write(fd, path+path+path) + os.close(fd) + + fd = os.open(path, os.O_RDWR, 0777) + maxread = 1000 + r = os.read(fd, maxread) + os.close(fd) + + return len(r) + + if os.path.exists(path): + os.unlink(path) + f = compile_function(openwriteclose_openreadclose, []) + result = f() + assert os.path.exists(path) + assert open(path).read() == path * 3 + assert result is len(path) * 3 + +# following from translator/c/test/test_extfunc.py Revision: 15320 (jul 29th 2005) + +def test_os_stat(): + if sys.maxint != 2**31-1: + py.test.skip("WIP on 64 bit architectures") + filename = str(py.magic.autopath()) + def call_stat0(): + st = os.stat(filename) + return st[0] + def call_stat1(): + st = os.stat(filename) + return st[1] + def call_stat2(): + st = os.stat(filename) + return st[2] + f0 = compile_function(call_stat0, []) + f1 = compile_function(call_stat1, []) + f2 = compile_function(call_stat2, []) + st = os.stat(filename) + assert f0() == st[0] + assert f1() == st[1] + assert f2() == st[2] + +def test_os_fstat(): + if sys.maxint != 2**31-1: + py.test.skip("WIP on 64 bit architectures") + filename = str(py.magic.autopath()) + def call_fstat0(): + fd = os.open(filename, os.O_RDONLY, 0777) + st = os.fstat(fd) + os.close(fd) + return st[0] + def call_fstat1(): + fd = os.open(filename, os.O_RDONLY, 0777) + st = os.fstat(fd) + os.close(fd) + return st[1] + def call_fstat2(): + fd = os.open(filename, os.O_RDONLY, 0777) + st = os.fstat(fd) + os.close(fd) + return st[2] + f0 = compile_function(call_fstat0, []) + f1 = compile_function(call_fstat1, []) + f2 = compile_function(call_fstat2, []) + st = os.stat(filename) + assert f0() == st[0] + assert f1() == st[1] + assert f2() == st[2] + +def test_os_getcwd(): + cwd = os.getcwd() + def does_stuff(): + return os.getcwd() == cwd + f1 = compile_function(does_stuff, []) + + assert f1() + +def test_os_path_exists(): + if sys.maxint != 2**31-1: + py.test.skip("WIP on 64 bit architectures") + tmpfile = str(udir.join('test_os_path_exists.TMP')) + def fn(): + return os.path.exists(tmpfile) + f = compile_function(fn, []) + open(tmpfile, 'w').close() + assert f() == True + os.unlink(tmpfile) + assert f() == False + +def test_dynamic_string_null_termination(): + if sys.maxint != 2**31-1: + py.test.skip("WIP on 64 bit architectures") + # forces malloc / versus pbc for NUL testing of C string + tmpfile = str(udir.join('test_os_path_exists.TMP')) + def fn(l): + filename = tmpfile[:l] + return os.path.exists(filename) + f = compile_function(fn, [r_uint]) + open(tmpfile, 'w').close() + lfile = len(tmpfile) + assert f(lfile) == True + assert f(lfile-2) == False + +def test_os_path_isdir(): + if sys.maxint != 2**31-1: + py.test.skip("WIP on 64 bit architectures") + directory = "./." + def fn(): + return os.path.isdir(directory) + f = compile_function(fn, []) + assert f() == True + directory = "some/random/name" + def fn(): + return os.path.isdir(directory) + f = compile_function(fn, []) + assert f() == False + +def test_os_isatty(): + def call_isatty(fd): + return os.isatty(fd) + f = compile_function(call_isatty, [int], isolate_hint=False) + assert f(0) == os.isatty(0) + assert f(1) == os.isatty(1) + assert f(2) == os.isatty(2) + + +def test_os_chmod(): + tmpfile = str(udir.join('test_os_chmod.txt')) + f = open(tmpfile, 'w') + f.close() + def does_stuff(mode): + os.chmod(tmpfile, mode) + return 0 + f1 = compile_function(does_stuff, [int]) + f1(0000) + assert os.stat(tmpfile).st_mode & 0777 == 0000 + f1(0644) + assert os.stat(tmpfile).st_mode & 0777 == 0644 + +def test_os_rename(): + tmpfile1 = str(udir.join('test_os_rename_1.txt')) + tmpfile2 = str(udir.join('test_os_rename_2.txt')) + f = open(tmpfile1, 'w') + f.close() + def does_stuff(): + os.rename(tmpfile1, tmpfile2) + return 0 + f1 = compile_function(does_stuff, []) + f1() + assert os.path.exists(tmpfile2) + assert not os.path.exists(tmpfile1) + +if hasattr(os, 'getpid'): + def test_os_getpid(): + def does_stuff(): + return os.getpid() + f1 = compile_function(does_stuff, [], isolate_hint=False) + res = f1() + assert res == os.getpid() + +if hasattr(os, 'link'): + def test_links(): + import stat + tmpfile1 = str(udir.join('test_links_1.txt')) + tmpfile2 = str(udir.join('test_links_2.txt')) + tmpfile3 = str(udir.join('test_links_3.txt')) + f = open(tmpfile1, 'w') + f.close() + def does_stuff(): + os.symlink(tmpfile1, tmpfile2) + os.link(tmpfile1, tmpfile3) + assert os.readlink(tmpfile2) == tmpfile1 + flag= 0 + st = os.lstat(tmpfile1) + flag = flag*10 + stat.S_ISREG(st[0]) + flag = flag*10 + stat.S_ISLNK(st[0]) + st = os.lstat(tmpfile2) + flag = flag*10 + stat.S_ISREG(st[0]) + flag = flag*10 + stat.S_ISLNK(st[0]) + st = os.lstat(tmpfile3) + flag = flag*10 + stat.S_ISREG(st[0]) + flag = flag*10 + stat.S_ISLNK(st[0]) + return flag + f1 = compile_function(does_stuff, []) + res = f1() + assert res == 100110 + assert os.path.islink(tmpfile2) + assert not os.path.islink(tmpfile3) +if hasattr(os, 'fork'): + def test_fork(): + def does_stuff(): + pid = os.fork() + if pid == 0: # child + os._exit(4) + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + return status1 + f1 = compile_function(does_stuff, []) + status1 = f1() + assert os.WIFEXITED(status1) + assert os.WEXITSTATUS(status1) == 4 + +posix = __import__(os.name) +if hasattr(posix, 'execv'): + def test_execv(): + py.test.skip("not working yet") + filename = str(udir.join('test_execv.txt')) + executable = sys.executable + def does_stuff(): + progname = str(executable) + l = ['', ''] + l[0] = progname + l[1] = "-c" + l.append('open("%s","w").write("1")' % filename) + pid = os.fork() + if pid == 0: + os.execv(progname, l) + else: + os.waitpid(pid, 0) + return 1 + func = compile_function(does_stuff, []) + func() + assert open(filename).read() == "1" + + def test_execv_raising(): + py.test.skip("not working yet") + def does_stuff(): + l = [] + l.append("asddsadw32eewdfwqdqwdqwd") + try: + os.execv(l[0], l) + except OSError: + return 1 + else: + return 0 + func = compile_function(does_stuff, []) + res = func() + assert res == 1 + + def test_execve(): + py.test.skip("not working yet") + filename = str(udir.join('test_execve.txt')) + executable = sys.executable + def does_stuff(): + progname = executable + l = [] + l.append(progname) + l.append("-c") + l.append('import os; open("%s", "w").write(os.environ["STH"])' % filename) + env = {} + env["STH"] = "42" + env["sthelse"] = "a" + pid = os.fork() + if pid == 0: + os.execve(progname, l, env) + else: + os.waitpid(pid, 0) + return 1 + func = compile_function(does_stuff, []) + func() + assert open(filename).read() == "42" + +def test_os_unlink(): + tmpfile = str(udir.join('test_os_path_exists.TMP')) + def fn(): + os.unlink(tmpfile) + return 0 + f = compile_function(fn, []) + open(tmpfile, 'w').close() + fn() + assert not os.path.exists(tmpfile) + +def test_chdir(): + path = '..' + def does_stuff(): + os.chdir(path) + return 0 + f1 = compile_function(does_stuff, []) + curdir = os.getcwd() + try: + os.chdir() + except: pass # toplevel + def does_stuff2(): + os.chdir(curdir) + return 0 + f1 = compile_function(does_stuff2, []) + f1() + assert curdir == os.getcwd() + +def test_mkdir_rmdir(): + path = str(udir.join('test_mkdir_rmdir')) + def does_stuff(delete): + if delete: + os.rmdir(path) + else: + os.mkdir(path, 0777) + return 0 + f1 = compile_function(does_stuff, [bool]) + f1(False) + assert os.path.exists(path) and os.path.isdir(path) + f1(True) + assert not os.path.exists(path) + +def test_pipe_dup_dup2(): + def does_stuff(): + a, b = os.pipe() + c = os.dup(a) + d = os.dup(b) + assert a != b + assert a != c + assert a != d + assert b != c + assert b != d + assert c != d + os.close(c) + os.dup2(d, c) + e, f = os.pipe() + assert e != a + assert e != b + assert e != c + assert e != d + assert f != a + assert f != b + assert f != c + assert f != d + assert f != e + os.close(a) + os.close(b) + os.close(c) + os.close(d) + os.close(e) + os.close(f) + return 42 + f1 = compile_function(does_stuff, []) + res = f1() + assert res == 42 Modified: pypy/dist/pypy/translator/llvm/test/test_rtagged.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rtagged.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rtagged.py Wed Sep 12 12:05:49 2007 @@ -1,6 +1,10 @@ +import py + import sys, os from pypy.rlib.objectmodel import UnboxedValue +from pypy.translator.llvm.test.runtest import * + class A(object): __slots__ = () def meth(self, x): @@ -62,13 +66,11 @@ # ____________________________________________________________ # only with Boehm so far -from pypy.translator.llvm.test import runtest from pypy.translator.interactive import Translation from pypy import conftest def test_tagged_boehm(): - runtest.llvm_test() - runtest.gcc3_test() + py.test.skip("broken as test need rffi") t = Translation(entry_point, standalone=True, gc='boehm') try: exename = t.compile_llvm() Added: pypy/dist/pypy/translator/llvm/test/test_threading.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_threading.py Wed Sep 12 12:05:49 2007 @@ -0,0 +1,88 @@ +import sys + +import py +py.test.skip("threading is broken") + +import thread + +from pypy.translator.llvm.test.runtest import * + +def test_lock(): + def fn(): + l = thread.allocate_lock() + ok1 = l.acquire(True) + ok2 = l.acquire(False) + l.release() + ok2_and_a_half = False + try: + l.release() + except thread.error: + ok2_and_a_half = True + ok3 = l.acquire(False) + return ok1 and not ok2 and ok2_and_a_half and ok3 + f = compile_function(fn, []) + res = f() + assert res + +def test_simple_start_new_thread(): + class Arg: + pass + def mythreadedfunction(arg): + assert arg.value == 42 + def myotherthreadedfunction(arg): + assert arg.value == 43 + a42 = Arg() + a42.value = 42 + a43 = Arg() + a43.value = 43 + def fn(i): + thread.start_new_thread(mythreadedfunction, (a42,)) + thread.start_new_thread(myotherthreadedfunction, (a43,)) + if i == 1: + x = mythreadedfunction + a = a42 + else: + x = myotherthreadedfunction + a = a43 + thread.start_new_thread(x, (a,)) + return 42 + f = compile_function(fn, [int]) + res = f(1) + assert res == 42 + +def test_start_new_thread(): + class Arg: + pass + a = Arg() + a.x = 5 + def mythreadedfunction(arg): + arg.x += 37 + arg.myident = thread.get_ident() + arg.lock.release() + def fn(): + a.lock = thread.allocate_lock() + a.lock.acquire(True) + ident = thread.start_new_thread(mythreadedfunction, (a,)) + assert ident != thread.get_ident() + a.lock.acquire(True) # wait for the thread to finish + assert a.myident == ident + return a.x + f = compile_function(fn, []) + res = f() + assert res == 42 + +def test_prebuilt_lock(): + lock0 = thread.allocate_lock() + lock1 = thread.allocate_lock() + lock1.acquire() + def fn(i): + lock = [lock0, lock1][i] + ok = lock.acquire(False) + if ok: lock.release() + return ok + f = compile_function(fn, [int]) + res = f(0) + assert res == True + res = f(1) + assert res == False + From antocuni at codespeak.net Wed Sep 12 12:17:43 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 12:17:43 +0200 (CEST) Subject: [pypy-svn] r46493 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070912101743.45DDB80F6@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 12:17:42 2007 New Revision: 46493 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Log: ll_os_lseek for jvm Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Wed Sep 12 12:17:42 2007 @@ -10,6 +10,7 @@ public abstract void write(String buffer); public abstract String read(int count); public abstract void close(); + public abstract RandomAccessFile getFile(); } class PrintStreamWrapper extends FileWrapper @@ -36,6 +37,11 @@ { ll_os.throwOSError(PyPy.EBADF, "Cannot close stdout or stderr"); } + + public RandomAccessFile getFile() + { + return null; + } } class InputStreamWrapper extends FileWrapper @@ -69,6 +75,11 @@ { ll_os.throwOSError(PyPy.EBADF, "Cannot close stdin"); } + + public RandomAccessFile getFile() + { + return null; + } } class RandomAccessFileWrapper extends FileWrapper @@ -125,6 +136,11 @@ ll_os.throwOSError(PyPy.EIO, e.getMessage()); } } + + public RandomAccessFile getFile() + { + return this.file; + } } @@ -147,6 +163,10 @@ private static final int S_IFDIR = 16384; private static final int S_IFREG = 32768; + private static final int SEEK_SET = 0; + private static final int SEEK_CUR = 1; + private static final int SEEK_END = 2; + private static int fdcount; private static Map FileDescriptors = new HashMap(); private static Map ErrorMessages = new HashMap(); @@ -240,6 +260,36 @@ return ll_os_read(fd, (int)count); } + public static long ll_os_lseek(int fd, long offset, int whence) + { + FileWrapper wrapper = getfd(fd); + RandomAccessFile file = wrapper.getFile(); + if (file == null) + throwOSError(PyPy.ESPIPE, "Illegal seek"); + + long pos = 0; + try { + switch(whence) + { + case SEEK_SET: + pos = offset; + break; + case SEEK_CUR: + pos = file.getFilePointer() + offset; + break; + case SEEK_END: + pos = file.length() + offset; + break; + } + file.seek(pos); + } + catch(IOException e) { + throwOSError(PyPy.ESPIPE, e.getMessage()); + } + + return pos; + } + public static StatResult ll_os_lstat(String path) { return ll_os_stat(path); // XXX From rxe at codespeak.net Wed Sep 12 12:19:03 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 12 Sep 2007 12:19:03 +0200 (CEST) Subject: [pypy-svn] r46494 - pypy/dist/pypy/translator/llvm/module Message-ID: <20070912101903.64D8D80F6@code0.codespeak.net> Author: rxe Date: Wed Sep 12 12:19:02 2007 New Revision: 46494 Modified: pypy/dist/pypy/translator/llvm/module/protos.h Log: can get rid of this hack now Modified: pypy/dist/pypy/translator/llvm/module/protos.h ============================================================================== --- pypy/dist/pypy/translator/llvm/module/protos.h (original) +++ pypy/dist/pypy/translator/llvm/module/protos.h Wed Sep 12 12:19:02 2007 @@ -8,13 +8,6 @@ #define RPyRaiseSimpleException(exctype, errormsg) raise##exctype(errormsg) -// XXX generated by rpython - argggh have to feed in prototypes -RPyFREXP_RESULT *ll_frexp_result(double, int); -RPyMODF_RESULT *ll_modf_result(double, double); -RPySTAT_RESULT *ll_stat_result(int, int, int, int, int, int, int, int, int, int); -RPyPIPE_RESULT *ll_pipe_result(int, int); -RPyWAITPID_RESULT *ll_waitpid_result(long, long); - void RPYTHON_RAISE_OSERROR(int error); #ifdef RPyListOfString RPyListOfString *_RPyListOfString_New(long); From antocuni at codespeak.net Wed Sep 12 12:24:05 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 12:24:05 +0200 (CEST) Subject: [pypy-svn] r46495 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070912102405.487E080C3@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 12:24:03 2007 New Revision: 46495 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Log: oops! swap stdin and stdout Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Wed Sep 12 12:24:03 2007 @@ -172,8 +172,8 @@ private static Map ErrorMessages = new HashMap(); static { - FileDescriptors.put(new Integer(0), new PrintStreamWrapper(System.out)); - FileDescriptors.put(new Integer(1), new InputStreamWrapper(System.in)); + FileDescriptors.put(new Integer(0), new InputStreamWrapper(System.in)); + FileDescriptors.put(new Integer(1), new PrintStreamWrapper(System.out)); FileDescriptors.put(new Integer(2), new PrintStreamWrapper(System.err)); fdcount = 2; } From rxe at codespeak.net Wed Sep 12 12:33:08 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 12 Sep 2007 12:33:08 +0200 (CEST) Subject: [pypy-svn] r46496 - pypy/dist/pypy/translator/llvm/module Message-ID: <20070912103308.A345980FB@code0.codespeak.net> Author: rxe Date: Wed Sep 12 12:33:08 2007 New Revision: 46496 Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c Log: well chopping about 10 lines allows 95% of the tests to run again (using llvm 1.9 not-via-c) Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Wed Sep 12 12:33:08 2007 @@ -4,16 +4,17 @@ #ifdef _RPyListOfString_New /* :-( */ # define HAVE_RPY_LIST_OF_STRING #endif + #include "c/src/thread.h" -#include "c/src/ll_os.h" +//#include "c/src/ll_os.h" #include "c/src/ll_math.h" #include "c/src/ll_time.h" #include "c/src/ll_strtod.h" -#include "c/src/ll_thread.h" +//#include "c/src/ll_thread.h" #include "c/src/stack.h" // setup code for ThreadLock Opaque types -char *RPyOpaque_LLVM_SETUP_ThreadLock(struct RPyOpaque_ThreadLock *lock, +/*char *RPyOpaque_LLVM_SETUP_ThreadLock(struct RPyOpaque_ThreadLock *lock, int initially_locked) { struct RPyOpaque_ThreadLock tmp = RPyOpaque_INITEXPR_ThreadLock; @@ -27,6 +28,7 @@ } return NULL; } +*/ // raw malloc code char *raw_malloc(long size) { From rxe at codespeak.net Wed Sep 12 12:41:52 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 12 Sep 2007 12:41:52 +0200 (CEST) Subject: [pypy-svn] r46497 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20070912104152.375628102@code0.codespeak.net> Author: rxe Date: Wed Sep 12 12:41:51 2007 New Revision: 46497 Removed: pypy/dist/pypy/translator/llvm/pyxwrapper.py Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/module/genexterns.c Log: kill pyrex dependency Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Wed Sep 12 12:41:51 2007 @@ -4,12 +4,66 @@ import py from pypy.translator.llvm.log import log -from pypy.translator.llvm.pyxwrapper import write_pyx_wrapper from pypy.translator.tool import stdoutcapture -from pypy.translator.tool.cbuild import make_c_from_pyxfile import distutils.sysconfig +def write_ctypes_module(genllvm, dllname, targetpath): + template = """ +import ctypes +from os.path import join, dirname, realpath +_c = ctypes.CDLL(join(dirname(realpath(__file__)), "%(dllname)s")) + +_setup = False + +class LLVMException(Exception): + pass + +%(name)s = _c.__entrypoint__%(name)s +%(name)s.argtypes = %(args)s +%(name)s.restype = %(returntype)s + +%(name)s_raised = _c.__entrypoint__raised_LLVMException +%(name)s_raised.argtypes = [] +%(name)s_raised.restype = ctypes.c_int + +GC_get_heap_size_wrapper = _c.GC_get_heap_size +GC_get_heap_size_wrapper.argtypes = [] +GC_get_heap_size_wrapper.restype = ctypes.c_int + +startup_code = _c.ctypes_RPython_StartupCode +startup_code.argtypes = [] +startup_code.restype = ctypes.c_int + +def %(name)s_wrapper(*args): + global _setup + if not _setup: + if not startup_code(): + raise LLVMException("Failed to startup") + _setup = True + result = %(name)s(*args) + if %(name)s_raised(): + raise LLVMException("Exception raised") + return result +""" + + import ctypes + from pypy.rpython.lltypesystem import lltype + + TO_CTYPES = {lltype.Bool: "ctypes.c_int", + lltype.Float: "ctypes.c_double", + lltype.Char: "ctypes.c_char", + lltype.Signed: "ctypes.c_int", + lltype.Unsigned: "ctypes.c_uint" + } + name = genllvm.entrynode.ref.strip("%") + + g = genllvm.entrynode.graph + returntype = TO_CTYPES[g.returnblock.inputargs[0].concretetype] + inputargtypes = [TO_CTYPES[a.concretetype] for a in g.startblock.inputargs] + args = '[%s]' % ", ".join(inputargtypes) + targetpath.write(template % locals()) + def llvm_is_on_path(): if py.path.local.sysfind("llvm-as") is None or \ py.path.local.sysfind("llvm-gcc") is None: @@ -45,22 +99,22 @@ gcc_version = lambda: _exe_version2('gcc') llvm_gcc_version = lambda: _exe_version2('llvm-gcc') -def compile_module(module, source_files, object_files, library_files): +# def compile_module(module, source_files, object_files, library_files): - open("%s_setup.py" % module, "w").write(str(py.code.Source( - ''' - from distutils.core import setup - from distutils.extension import Extension - setup(name="%(module)s", - ext_modules = [Extension( - name = "%(module)s", - sources = %(source_files)s, - libraries = %(library_files)s, - extra_objects = %(object_files)s)]) - ''' % locals()))) - cmd ="python %s_setup.py build_ext --inplace --force" % module - log.build(cmd) - py.process.cmdexec(cmd) +# open("%s_setup.py" % module, "w").write(str(py.code.Source( +# ''' +# from distutils.core import setup +# from distutils.extension import Extension +# setup(name="%(module)s", +# ext_modules = [Extension( +# name = "%(module)s", +# sources = %(source_files)s, +# libraries = %(library_files)s, +# extra_objects = %(object_files)s)]) +# ''' % locals()))) +# cmd ="python %s_setup.py build_ext --inplace --force" % module +# log.build(cmd) +# py.process.cmdexec(cmd) class Builder(object): @@ -131,37 +185,34 @@ library_files = self.genllvm.db.gcpolicy.gc_libraries() gc_libs = ' '.join(['-l' + lib for lib in library_files]) - object_files = ["-L/sw/lib"] if sys.platform == 'darwin': libdir = '/sw/lib' gc_libs_path = '-L%s -ldl' % libdir else: gc_libs_path = '-static' - use_gcc = True #self.genllvm.config.translation.llvm_via_c + dllname = "%s.so" % b + + use_gcc = False #self.genllvm.config.translation.llvm_via_c if not use_gcc: self.cmds.append("llc -relocation-model=pic %s.bc -f -o %s.s" % (b, b)) self.cmds.append("as %s.s -o %s.o" % (b, b)) - object_files.append("%s.o" % b) + else: self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (b, b)) self.cmds.append("gcc %s.c -c -O2" % b) - object_files.append("%s.o" % b) + + self.cmds.append("gcc -O3 %s.o %s %s -lm -bundle -o %s" % (b, gc_libs_path, gc_libs, dllname)) try: self.execute_cmds() - # use pyrex to create module for CPython - basename = self.genllvm.filename.purebasename + '_wrapper.pyx' - pyxfile = self.genllvm.filename.new(basename = basename) - write_pyx_wrapper(self.genllvm, pyxfile) - - modname = pyxfile.purebasename - source_files = ["%s.c" % modname] - - make_c_from_pyxfile(pyxfile) + # use ctypes to create module for CPython + basename = self.genllvm.filename.purebasename + '_wrapper.py' + modfilename = self.genllvm.filename.new(basename = basename) + write_ctypes_module(self.genllvm, dllname, modfilename) - compile_module(modname, source_files, object_files, library_files) + modname = modfilename.purebasename finally: lastdir.chdir() Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Wed Sep 12 12:41:51 2007 @@ -48,7 +48,9 @@ cmd = "llvm-gcc %s %s %s -S %s.c -o %s.ll 2>&1" % ( include_path, includes, emit_llvm, plain, plain) - os.system(cmd) + if os.system(cmd) != 0: + raise Exception("Failed to run '%s'") + llcode = open(plain + '.ll').read() # strip lines @@ -78,7 +80,7 @@ funcname , s = s.split('(', 1) funcnames[funcname] = True if line.find("internal") == -1: - if funcname not in ["%main", "%Pyrex_RPython_StartupCode"]: + if funcname not in ["%main", "%ctypes_RPython_StartupCode"]: internal = 'internal ' line = '%s%s %s' % (internal, DEFAULT_CCONV, line,) ll_lines.append(line) Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Wed Sep 12 12:41:51 2007 @@ -24,9 +24,6 @@ def gc_libraries(self): return [] - def pyrex_code(self): - return '' - def get_count(self, inc=False): if inc: self.n_malloced = self.n_malloced + 1 @@ -113,14 +110,6 @@ def gc_libraries(self): return ['gc', 'pthread'] - def pyrex_code(self): - return ''' -cdef extern int GC_get_heap_size() - -def GC_get_heap_size_wrapper(): - return GC_get_heap_size() -''' - def _zeromalloc(self, codewriter, targetvar, size=1, atomic=False, exc_flag=False): """ assumes malloc of word size """ Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Wed Sep 12 12:41:51 2007 @@ -97,10 +97,10 @@ #else -int Pyrex_RPython_StartupCode() { +int ctypes_RPython_StartupCode() { - char *error = RPython_StartupCode(); - if (error != NULL) { + char *errmsg = RPython_StartupCode(); + if (errmsg != NULL) { return 0; } From antocuni at codespeak.net Wed Sep 12 13:46:34 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 13:46:34 +0200 (CEST) Subject: [pypy-svn] r46498 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070912114634.A433A80F8@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 13:46:33 2007 New Revision: 46498 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Log: ll_os_isatty Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Wed Sep 12 13:46:33 2007 @@ -312,6 +312,16 @@ return text.length(); } + public static boolean ll_os_isatty(int x) + { + // XXX: this is not the right behaviour, but it's needed + // to have the interactive interpreter working + if (x == 0 || x == 1 || x == 2) + return true; + else + return false; + } + public static ArrayList ll_os_envitems() { return new ArrayList(); // XXX From antocuni at codespeak.net Wed Sep 12 13:56:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 13:56:00 +0200 (CEST) Subject: [pypy-svn] r46499 - in pypy/dist/pypy/translator/jvm: src/pypy test Message-ID: <20070912115600.4C82F80F8@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 13:55:57 2007 New Revision: 46499 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java pypy/dist/pypy/translator/jvm/test/test_builtin.py Log: test_os_open_write passes! Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Wed Sep 12 13:55:57 2007 @@ -101,7 +101,7 @@ ll_os.throwOSError(PyPy.EBADF, "Cannot write to this fd"); try { - this.file.writeChars(buffer); + this.file.writeBytes(buffer); } catch(IOException e) { ll_os.throwOSError(PyPy.EIO, e.getMessage()); Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Wed Sep 12 13:55:57 2007 @@ -3,13 +3,16 @@ from pypy.translator.oosupport.test_template.builtin import BaseTestBuiltin, BaseTestTime from pypy.translator.jvm.test.runtest import JvmTest -class TestJavaBuiltin(JvmTest, BaseTestBuiltin): +def skip_win(): + import platform + if platform.system() == 'Windows': + py.test.skip("Doesn't work on Windows, yet") - def test_os_open_write(self): - py.test.skip("ll_os_open is not currently implemented in the Jvm backed") +class TestJavaBuiltin(JvmTest, BaseTestBuiltin): def test_os_write_magic(self): - py.test.skip("ll_os_open is not currently implemented in the Jvm backed") + skip_win() + BaseTestBuiltin.test_os_write_magic(self) def test_builtin_math_frexp(self): py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") From antocuni at codespeak.net Wed Sep 12 14:05:47 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 12 Sep 2007 14:05:47 +0200 (CEST) Subject: [pypy-svn] r46500 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20070912120547.3ED3D80F8@code0.codespeak.net> Author: antocuni Date: Wed Sep 12 14:05:46 2007 New Revision: 46500 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Log: fix a bug when reading from stdin Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Wed Sep 12 14:05:46 2007 @@ -63,7 +63,9 @@ try { byte[] buf = new byte[count]; int n = stream.read(buf, 0, count); - return new String(buf); + if (n == -1) + return ""; // XXX: is it right? + return new String(buf, 0, n); } catch(IOException e) { ll_os.throwOSError(PyPy.EIO, e.getMessage()); From arigo at codespeak.net Wed Sep 12 18:01:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 Sep 2007 18:01:43 +0200 (CEST) Subject: [pypy-svn] r46506 - in pypy: branch/cleanup-weakref dist/pypy/annotation dist/pypy/module dist/pypy/rlib dist/pypy/rpython dist/pypy/rpython/lltypesystem dist/pypy/rpython/memory dist/pypy/translator dist/pypy/translator/c dist/pypy/translator/cli dist/pypy/translator/jvm dist/pypy/translator/llvm/test dist/pypy/translator/oosupport/test_template dist/pypy/translator/stackless Message-ID: <20070912160143.21B31810E@code0.codespeak.net> Author: arigo Date: Wed Sep 12 18:01:42 2007 New Revision: 46506 Added: pypy/dist/pypy/annotation/ - copied from r46505, pypy/branch/cleanup-weakref/pypy/annotation/ pypy/dist/pypy/module/ - copied from r46505, pypy/branch/cleanup-weakref/pypy/module/ pypy/dist/pypy/rlib/ - copied from r46505, pypy/branch/cleanup-weakref/pypy/rlib/ pypy/dist/pypy/rpython/lltypesystem/ - copied from r46505, pypy/branch/cleanup-weakref/pypy/rpython/lltypesystem/ pypy/dist/pypy/rpython/memory/ - copied from r46505, pypy/branch/cleanup-weakref/pypy/rpython/memory/ pypy/dist/pypy/rpython/raddress.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/rpython/raddress.py pypy/dist/pypy/rpython/rbuiltin.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rmodel.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/rpython/rmodel.py pypy/dist/pypy/rpython/rweakref.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/rpython/rweakref.py pypy/dist/pypy/translator/c/ - copied from r46505, pypy/branch/cleanup-weakref/pypy/translator/c/ pypy/dist/pypy/translator/cli/metavm.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/translator/cli/metavm.py pypy/dist/pypy/translator/cli/opcodes.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/exceptiontransform.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/translator/exceptiontransform.py pypy/dist/pypy/translator/jvm/metavm.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/translator/jvm/metavm.py pypy/dist/pypy/translator/jvm/opcodes.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/llvm/test/test_lladdresses.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/translator/llvm/test/test_lladdresses.py pypy/dist/pypy/translator/oosupport/test_template/objectmodel.py - copied unchanged from r46505, pypy/branch/cleanup-weakref/pypy/translator/oosupport/test_template/objectmodel.py pypy/dist/pypy/translator/stackless/ - copied from r46505, pypy/branch/cleanup-weakref/pypy/translator/stackless/ Removed: pypy/branch/cleanup-weakref/ Log: (cfbolz, arigo) merge the cleanup-weakref branch. Weakrefs are now malloced objects on their own, and the ll code manipulates (regular) pointers to such objects. From tismer at codespeak.net Wed Sep 12 18:49:17 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 12 Sep 2007 18:49:17 +0200 (CEST) Subject: [pypy-svn] r46507 - pypy/dist/pypy/lib Message-ID: <20070912164917.015E080E2@code0.codespeak.net> Author: tismer Date: Wed Sep 12 18:49:17 2007 New Revision: 46507 Modified: pypy/dist/pypy/lib/stackless.py Log: space/tab quirks Modified: pypy/dist/pypy/lib/stackless.py ============================================================================== --- pypy/dist/pypy/lib/stackless.py (original) +++ pypy/dist/pypy/lib/stackless.py Wed Sep 12 18:49:17 2007 @@ -402,8 +402,8 @@ def insert(self): if self.blocked: raise RuntimeError, "You cannot run a blocked tasklet" - if not self.alive: - raise RuntimeError, "You cannot run an unbound(dead) tasklet" + if not self.alive: + raise RuntimeError, "You cannot run an unbound(dead) tasklet" _scheduler_append(self) def remove(self): @@ -411,7 +411,7 @@ raise RuntimeError, "You cannot remove a blocked tasklet." if self is getcurrent(): raise RuntimeError, "The current tasklet cannot be removed." - # not sure if I will revive this " Use t=tasklet().capture()" + # not sure if I will revive this " Use t=tasklet().capture()" _scheduler_remove(self) def __reduce__(self): From arigo at codespeak.net Wed Sep 12 19:14:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 Sep 2007 19:14:53 +0200 (CEST) Subject: [pypy-svn] r46508 - in pypy/dist/pypy: annotation rpython rpython/lltypesystem rpython/memory/gctransform translator/c/src Message-ID: <20070912171453.0962680D5@code0.codespeak.net> Author: arigo Date: Wed Sep 12 19:14:53 2007 New Revision: 46508 Modified: pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/memory/gctransform/boehm.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/translator/c/src/mem.h Log: Add cast_ptr_to_weakrefptr and back. See the comments in llmemory. It seems to be required for the framework gc to implement weakrefs in a way that is testable on top of the llinterpreter. Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Wed Sep 12 19:14:53 2007 @@ -591,9 +591,26 @@ "got %s" % (s_wref,)) return SomePtr(s_ptrtype.const) +def llcast_ptr_to_weakrefptr(s_ptr): + assert isinstance(s_ptr, SomePtr) + return SomePtr(llmemory.WeakRefPtr) + +def llcast_weakrefptr_to_ptr(s_ptrtype, s_wref): + if not (s_ptrtype.is_constant() and + isinstance(s_ptrtype.const, lltype.Ptr)): + raise Exception("cast_weakrefptr_to_ptr() arg 1 must be a constant " + "ptr type, got %s" % (s_ptrtype,)) + if not (isinstance(s_wref, SomePtr) and + s_wref.ll_ptrtype == llmemory.WeakRefPtr): + raise Exception("cast_weakrefptr_to_ptr() arg 2 must be a WeakRefPtr, " + "got %s" % (s_wref,)) + return SomePtr(s_ptrtype.const) + from pypy.rpython.lltypesystem import llmemory BUILTIN_ANALYZERS[llmemory.weakref_create] = llweakref_create BUILTIN_ANALYZERS[llmemory.weakref_deref ] = llweakref_deref +BUILTIN_ANALYZERS[llmemory.cast_ptr_to_weakrefptr] = llcast_ptr_to_weakrefptr +BUILTIN_ANALYZERS[llmemory.cast_weakrefptr_to_ptr] = llcast_weakrefptr_to_ptr #________________________________ # non-gc objects Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Wed Sep 12 19:14:53 2007 @@ -709,6 +709,13 @@ return llmemory.weakref_deref(PTRTYPE, obj) op_weakref_deref.need_result_type = True + def op_cast_ptr_to_weakrefptr(self, obj): + return llmemory.cast_ptr_to_weakrefptr(obj) + + def op_cast_weakrefptr_to_ptr(self, PTRTYPE, obj): + return llmemory.cast_weakrefptr_to_ptr(PTRTYPE, obj) + op_cast_weakrefptr_to_ptr.need_result_type = True + def op_gc__collect(self): import gc gc.collect() Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Wed Sep 12 19:14:53 2007 @@ -548,6 +548,38 @@ # a prebuilt pointer to a dead low-level weakref dead_wref = _wref(None)._as_ptr() +# The rest is to support the GC transformers: they can use it to build +# an explicit weakref object with some structure and then "hide" the +# result by casting it to a WeakRefPtr, and "reveal" it again. In other +# words, weakref_create and weakref_deref are operations that exist only +# before the GC transformation, whereas the two cast operations below +# exist only after. They are implemented here only to allow GC +# transformers to be tested on top of the llinterpreter. +def cast_ptr_to_weakrefptr(ptr): + if ptr: + return _gctransformed_wref(ptr)._as_ptr() + else: + return lltype.nullptr(WeakRef) + +def cast_weakrefptr_to_ptr(PTRTYPE, pwref): + assert lltype.typeOf(pwref) == WeakRefPtr + if pwref: + assert isinstance(pwref._obj, _gctransformed_wref) + assert PTRTYPE == lltype.typeOf(pwref._obj._ptr) + return pwref._obj._ptr + else: + return lltype.nullptr(PTRTYPE.TO) + +class _gctransformed_wref(lltype._container): + _gckind = 'gc' + _TYPE = WeakRef + def __init__(self, ptr): + self._ptr = ptr + def __repr__(self): + return '<%s>' % (self,) + def __str__(self): + return 'gctransformed_wref(%s)' % (self._ptr,) + # ____________________________________________________________ def raw_malloc(size): Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Wed Sep 12 19:14:53 2007 @@ -406,6 +406,8 @@ # in the framework GC, so it should never cause a stack unwind. 'weakref_create': LLOp(canraise=(MemoryError,), sideeffects=False), 'weakref_deref': LLOp(sideeffects=False), + 'cast_ptr_to_weakrefptr': LLOp(canfold=True), # no-op, pure type hiding + 'cast_weakrefptr_to_ptr': LLOp(canfold=True), # no-op, pure type revealing # __________ stackless operation(s) __________ Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Wed Sep 12 19:14:53 2007 @@ -160,13 +160,14 @@ def ll_weakref_create(targetaddr): link = llop.boehm_malloc_atomic(llmemory.Address, sizeof_weakreflink) - link.address[0] = targetaddr + plink = llmemory.cast_adr_to_ptr(link, lltype.Ptr(WEAKLINK)) + plink[0] = targetaddr llop.boehm_disappearing_link(lltype.Void, link, targetaddr) - return llmemory.cast_adr_to_ptr(link, llmemory.WeakRefPtr) + return llmemory.cast_ptr_to_weakrefptr(plink) def ll_weakref_deref(wref): - link = llmemory.cast_ptr_to_adr(wref) - return link and link.address[0] + plink = llmemory.cast_weakrefptr_to_ptr(lltype.Ptr(WEAKLINK), wref) + return plink[0] def convert_weakref_to(targetptr): # Prebuilt weakrefs don't really need to be weak at all, @@ -176,6 +177,6 @@ if not targetptr: return empty_weaklink else: - link = lltype.malloc(WEAKLINK, immortal=True) - link[0] = llmemory.cast_ptr_to_adr(targetptr) - return link + plink = lltype.malloc(WEAKLINK, immortal=True) + plink[0] = llmemory.cast_ptr_to_adr(targetptr) + return plink Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Wed Sep 12 19:14:53 2007 @@ -610,9 +610,24 @@ hop.exception_cannot_occur() return hop.genop('weakref_deref', [v_wref], resulttype=c_ptrtype.value) +def rtype_cast_ptr_to_weakrefptr(hop): + vlist = hop.inputargs(hop.args_r[0]) + hop.exception_cannot_occur() + return hop.genop('cast_ptr_to_weakrefptr', vlist, + resulttype=llmemory.WeakRefPtr) + +def rtype_cast_weakrefptr_to_ptr(hop): + c_ptrtype, v_wref = hop.inputargs(lltype.Void, hop.args_r[1]) + assert v_wref.concretetype == llmemory.WeakRefPtr + hop.exception_cannot_occur() + return hop.genop('cast_weakrefptr_to_ptr', [v_wref], + resulttype=c_ptrtype.value) + BUILTIN_TYPER[weakref.ref] = rtype_weakref_create BUILTIN_TYPER[llmemory.weakref_create] = rtype_weakref_create BUILTIN_TYPER[llmemory.weakref_deref ] = rtype_weakref_deref +BUILTIN_TYPER[llmemory.cast_ptr_to_weakrefptr] = rtype_cast_ptr_to_weakrefptr +BUILTIN_TYPER[llmemory.cast_weakrefptr_to_ptr] = rtype_cast_weakrefptr_to_ptr # _________________________________________________________________ # non-gc objects Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Wed Sep 12 19:14:53 2007 @@ -117,3 +117,9 @@ } \ } #define OP_CPY_FREE(x) OP_RAW_FREE(x, /*nothing*/) + +/************************************************************/ +/* weakref support */ + +#define OP_CAST_PTR_TO_WEAKREFPTR(x, r) r = x +#define OP_CAST_WEAKREFPTR_TO_PTR(x, r) r = x From tismer at codespeak.net Wed Sep 12 19:46:05 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 12 Sep 2007 19:46:05 +0200 (CEST) Subject: [pypy-svn] r46509 - pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key Message-ID: <20070912174605.BCF1480B3@code0.codespeak.net> Author: tismer Date: Wed Sep 12 19:46:02 2007 New Revision: 46509 Removed: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.key/ Log: remove "Stackless Nuts And Bolts.key" as keynote is extremely svn hostile From tismer at codespeak.net Wed Sep 12 19:47:05 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 12 Sep 2007 19:47:05 +0200 (CEST) Subject: [pypy-svn] r46510 - pypy/extradoc/talk/pyconuk07 Message-ID: <20070912174705.6E34B80EE@code0.codespeak.net> Author: tismer Date: Wed Sep 12 19:47:04 2007 New Revision: 46510 Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.dmg (contents, props changed) Log: remove "Stackless Nuts And Bolts.key" as keynote is extremely svn hostile Added: pypy/extradoc/talk/pyconuk07/Stackless Nuts And Bolts.dmg ============================================================================== Binary file. No diff available. From tismer at codespeak.net Wed Sep 12 19:57:05 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 12 Sep 2007 19:57:05 +0200 (CEST) Subject: [pypy-svn] r46511 - pypy/dist/pypy/lib Message-ID: <20070912175705.673B380B3@code0.codespeak.net> Author: tismer Date: Wed Sep 12 19:57:03 2007 New Revision: 46511 Modified: pypy/dist/pypy/lib/stackless.py Log: thanks for finding this indentation bug. Maybe this prevents the bus error Modified: pypy/dist/pypy/lib/stackless.py ============================================================================== --- pypy/dist/pypy/lib/stackless.py (original) +++ pypy/dist/pypy/lib/stackless.py Wed Sep 12 19:57:03 2007 @@ -403,7 +403,7 @@ if self.blocked: raise RuntimeError, "You cannot run a blocked tasklet" if not self.alive: - raise RuntimeError, "You cannot run an unbound(dead) tasklet" + raise RuntimeError, "You cannot run an unbound(dead) tasklet" _scheduler_append(self) def remove(self): From cfbolz at codespeak.net Wed Sep 12 22:15:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Sep 2007 22:15:04 +0200 (CEST) Subject: [pypy-svn] r46519 - in pypy/dist/pypy: config rpython/memory rpython/memory/gctransform rpython/memory/test Message-ID: <20070912201504.CAA8D80FF@code0.codespeak.net> Author: cfbolz Date: Wed Sep 12 22:15:04 2007 New Revision: 46519 Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gcheader.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: support for weakrefs in the mark-and-sweep gc. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Wed Sep 12 22:15:04 2007 @@ -45,10 +45,9 @@ "exact_boehm"], "ref", requires={ "ref": [("translation.rweakref", False)], # XXX - "framework": [("translation.rweakref", False)], # XXX "none": [("translation.rweakref", False)], # XXX "stacklessgc": [("translation.stackless", True), - ("translation.rweakref", False)]}, # XXX + ]}, cmdline="--gc"), BoolOption("thread", "enable use of threading primitives", default=False, cmdline="--thread", Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Wed Sep 12 22:15:04 2007 @@ -65,7 +65,8 @@ fixed_size, varsize_item_sizes, varsize_offset_to_variable_part, varsize_offset_to_length, - varsize_offsets_to_gcpointers_in_var_part): + varsize_offsets_to_gcpointers_in_var_part, + weakpointer_offset): self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.offsets_to_gc_pointers = offsets_to_gc_pointers @@ -74,6 +75,7 @@ self.varsize_offset_to_variable_part = varsize_offset_to_variable_part self.varsize_offset_to_length = varsize_offset_to_length self.varsize_offsets_to_gcpointers_in_var_part = varsize_offsets_to_gcpointers_in_var_part + self.weakpointer_offset = weakpointer_offset def write_barrier(self, addr, addr_to, addr_struct): addr_to.address[0] = addr @@ -140,6 +142,9 @@ self.AddressLinkedList = AddressLinkedList self.malloced_objects = lltype.nullptr(self.HDR) self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) + # these are usually only the small bits of memory that make a + # weakref object + self.objects_with_weak_pointers = lltype.nullptr(self.HDR) self.get_roots = get_roots self.gcheaderbuilder = GCHeaderBuilder(self.HDR) # pools, for x_swap_pool(): @@ -149,6 +154,9 @@ # The exception is 'curpool' whose linked list of objects is in # 'self.malloced_objects' instead of in the header of 'curpool'. # POOL objects are never in the middle of a linked list themselves. + # XXX a likely cause for the current problems with pools is: + # not all objects live in malloced_objects, some also live in + # malloced_objects_with_finalizer and objects_with_weak_pointers self.curpool = lltype.nullptr(self.POOL) # 'poolnodes' is a linked list of all such linked lists. Each # linked list will usually start with a POOL object, but it can @@ -162,18 +170,23 @@ def malloc(self, typeid, length=0): size = self.fixed_size(typeid) needs_finalizer = bool(self.getfinalizer(typeid)) + contains_weakptr = self.weakpointer_offset(typeid) != -1 + assert needs_finalizer != contains_weakptr if self.is_varsize(typeid): + assert not contains_weakptr itemsize = self.varsize_item_sizes(typeid) offset_to_length = self.varsize_offset_to_length(typeid) ref = self.malloc_varsize(typeid, length, size, itemsize, - offset_to_length, True, needs_finalizer) + offset_to_length, True, needs_finalizer, + contains_weakptr) else: ref = self.malloc_fixedsize(typeid, size, True, needs_finalizer) # XXX lots of cast and reverse-cast around, but this malloc() # should eventually be killed return llmemory.cast_ptr_to_adr(ref) - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False): + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, + contains_weakptr=False): if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: self.collect() size_gc_header = self.gcheaderbuilder.size_gc_header @@ -192,6 +205,9 @@ if has_finalizer: hdr.next = self.malloced_objects_with_finalizer self.malloced_objects_with_finalizer = hdr + elif contains_weakptr: + hdr.next = self.objects_with_weak_pointers + self.objects_with_weak_pointers = hdr else: hdr.next = self.malloced_objects self.malloced_objects = hdr @@ -201,7 +217,8 @@ # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False): + def malloc_fixedsize_clear(self, typeid, size, can_collect, + has_finalizer=False, contains_weakptr=False): if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: self.collect() size_gc_header = self.gcheaderbuilder.size_gc_header @@ -221,6 +238,9 @@ if has_finalizer: hdr.next = self.malloced_objects_with_finalizer self.malloced_objects_with_finalizer = hdr + elif contains_weakptr: + hdr.next = self.objects_with_weak_pointers + self.objects_with_weak_pointers = hdr else: hdr.next = self.malloced_objects self.malloced_objects = hdr @@ -264,8 +284,9 @@ # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - def malloc_varsize_clear(self, typeid, length, size, itemsize, offset_to_length, - can_collect, has_finalizer=False): + def malloc_varsize_clear(self, typeid, length, size, itemsize, + offset_to_length, can_collect, + has_finalizer=False): if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: self.collect() size_gc_header = self.gcheaderbuilder.size_gc_header @@ -372,7 +393,41 @@ gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) hdr.typeid = hdr.typeid | 1 - + # go through the list of objects containing weak pointers + # and kill the links if they go to dead objects + # if the object itself is not marked, free it + hdr = self.objects_with_weak_pointers + surviving = lltype.nullptr(self.HDR) + while hdr: + typeid = hdr.typeid >> 1 + next = hdr.next + addr = llmemory.cast_ptr_to_adr(hdr) + size = self.fixed_size(typeid) + estimate = raw_malloc_usage(size_gc_header + size) + if hdr.typeid & 1: + typeid = hdr.typeid >> 1 + offset = self.weakpointer_offset(typeid) + hdr.typeid = hdr.typeid & (~1) + gc_info = llmemory.cast_ptr_to_adr(hdr) + weakref_obj = gc_info + size_gc_header + pointing_to = (weakref_obj + offset).address[0] + if pointing_to: + gc_info_pointing_to = pointing_to - size_gc_header + hdr_pointing_to = llmemory.cast_adr_to_ptr( + gc_info_pointing_to, self.HDRPTR) + # pointed to object will die + # XXX what to do if the object has a finalizer which resurrects + # the object? + if not hdr_pointing_to.typeid & 1: + (weakref_obj + offset).address[0] = NULL + hdr.next = surviving + surviving = hdr + curr_heap_size += estimate + else: + freed_size += estimate + raw_free(addr) + hdr = next + self.objects_with_weak_pointers = surviving # sweep: delete objects without del if they are not marked # unmark objects without del that are marked firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') Modified: pypy/dist/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcheader.py (original) +++ pypy/dist/pypy/rpython/memory/gcheader.py Wed Sep 12 22:15:04 2007 @@ -15,7 +15,11 @@ self.size_gc_header = llmemory.GCHeaderOffset(self) def header_of_object(self, gcptr): - return self.obj2header[gcptr._as_obj()] + # XXX hackhackhack + gcptr = gcptr._as_obj() + if isinstance(gcptr, llmemory._gctransformed_wref): + return self.obj2header[gcptr._ptr._as_obj()] + return self.obj2header[gcptr] def object_from_header(headerptr): return header2obj[headerptr._as_obj()] Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Sep 12 22:15:04 2007 @@ -52,6 +52,7 @@ ("ofstovar", lltype.Signed), ("ofstolength", lltype.Signed), ("varofstoptrs",lltype.Ptr(OFFSETS_TO_GC_PTR)), + ("weakptrofs", lltype.Signed), ) TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) @@ -79,6 +80,9 @@ def q_varsize_offsets_to_gcpointers_in_var_part(typeid): return gcdata.type_info_table[typeid].varofstoptrs + def q_weakpointer_offset(typeid): + return gcdata.type_info_table[typeid].weakptrofs + gcdata = GCData() # set up dummy a table, to be overwritten with the real one in finish() gcdata.type_info_table = lltype.malloc(GCData.TYPE_INFO_TABLE, 0, @@ -117,7 +121,8 @@ q_varsize_item_sizes, q_varsize_offset_to_variable_part, q_varsize_offset_to_length, - q_varsize_offsets_to_gcpointers_in_var_part) + q_varsize_offsets_to_gcpointers_in_var_part, + q_weakpointer_offset) bk = self.translator.annotator.bookkeeper @@ -178,9 +183,9 @@ self.incr_stack_ptr = None self.decr_stack_ptr = None self.save_addr_ptr = None - + self.weakref_deref_ptr = self.inittime_helper( + ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address) - classdef = bk.getuniqueclassdef(GCClass) s_gc = annmodel.SomeInstance(classdef) s_gcref = annmodel.SomePtr(llmemory.GCREF) @@ -188,13 +193,15 @@ GCClass.malloc_fixedsize.im_func, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), - annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, + annmodel.SomeBool(), annmodel.SomeBool(), + annmodel.SomeBool()], s_gcref, inline = False) self.malloc_fixedsize_clear_ptr = getfn( GCClass.malloc_fixedsize_clear.im_func, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), - annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, + annmodel.SomeBool(), annmodel.SomeBool(), + annmodel.SomeBool()], s_gcref, inline = False) ## self.malloc_varsize_ptr = getfn( ## GCClass.malloc_varsize.im_func, @@ -324,6 +331,7 @@ offsets = offsets_to_gc_pointers(TYPE) info["ofstoptrs"] = self.offsets2table(offsets, TYPE) info["finalyzer"] = self.finalizer_funcptr_for_type(TYPE) + info["weakptrofs"] = weakpointer_offset(TYPE) if not TYPE._is_varsize(): info["isvarsize"] = False info["fixedsize"] = llmemory.sizeof(TYPE) @@ -494,6 +502,9 @@ c_type_id = rmodel.inputconst(lltype.Signed, type_id) info = self.type_info_list[type_id] c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + c_has_finalizer = rmodel.inputconst( + lltype.Bool, bool(self.finalizer_funcptr_for_type(TYPE))) + if not op.opname.endswith('_varsize'): #malloc_ptr = self.malloc_fixedsize_ptr zero = flags.get('zero', False) @@ -501,7 +512,8 @@ malloc_ptr = self.malloc_fixedsize_clear_ptr else: malloc_ptr = self.malloc_fixedsize_ptr - args = [self.c_const_gc, c_type_id, c_size, c_can_collect] + args = [self.c_const_gc, c_type_id, c_size, c_can_collect, + c_has_finalizer, rmodel.inputconst(lltype.Bool, False)] else: v_length = op.args[-1] c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) @@ -512,11 +524,8 @@ ## else: ## malloc_ptr = self.malloc_varsize_clear_ptr args = [self.c_const_gc, c_type_id, v_length, c_size, - c_varitemsize, c_ofstolength, c_can_collect] - c_has_finalizer = rmodel.inputconst( - lltype.Bool, bool(self.finalizer_funcptr_for_type(TYPE))) - args.append(c_has_finalizer) - + c_varitemsize, c_ofstolength, c_can_collect, + c_has_finalizer] self.push_roots(hop) v_result = hop.genop("direct_call", [malloc_ptr] + args, resulttype=llmemory.GCREF) @@ -568,6 +577,42 @@ TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) + def gct_weakref_create(self, hop): + op = hop.spaceop + + type_id = self.get_type_id(WEAKREF) + + c_type_id = rmodel.inputconst(lltype.Signed, type_id) + info = self.type_info_list[type_id] + c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + malloc_ptr = self.malloc_fixedsize_ptr + c_has_finalizer = rmodel.inputconst(lltype.Bool, False) + c_has_weakptr = c_can_collect = rmodel.inputconst(lltype.Bool, True) + args = [self.c_const_gc, c_type_id, c_size, c_can_collect, + c_has_finalizer, c_has_weakptr] + + v_instance, = op.args + v_addr = hop.genop("cast_ptr_to_adr", [v_instance], + resulttype=llmemory.Address) + self.push_roots(hop) + v_result = hop.genop("direct_call", [malloc_ptr] + args, + resulttype=llmemory.GCREF) + v_result = hop.genop("cast_opaque_ptr", [v_result], + resulttype=WEAKREFPTR) + self.pop_roots(hop) + hop.genop("bare_setfield", + [v_result, rmodel.inputconst(lltype.Void, "weakptr"), v_addr]) + v_weakref = hop.genop("cast_ptr_to_weakrefptr", [v_result], + resulttype=llmemory.WeakRefPtr) + hop.cast_result(v_weakref) + + def gct_weakref_deref(self, hop): + v_wref, = hop.spaceop.args + v_addr = hop.genop("direct_call", + [self.weakref_deref_ptr, v_wref], + resulttype=llmemory.Address) + hop.cast_result(v_addr) + def push_alive_nopyobj(self, var, llops): pass @@ -625,6 +670,11 @@ offsets.append(0) return offsets +def weakpointer_offset(TYPE): + if TYPE == WEAKREF: + return llmemory.offsetof(WEAKREF, "weakptr") + return -1 + def gen_zero_gc_pointers(TYPE, v, llops): assert isinstance(TYPE, lltype.Struct) for name in TYPE._names: @@ -656,3 +706,29 @@ for i in range(len(v.items)): for a in gc_pointers_inside(v.items[i], adr + llmemory.itemoffsetof(t, i)): yield a + + +########## weakrefs ########## +# framework: weakref objects are small structures containing only an address + +WEAKREF = lltype.GcStruct("weakref", ("weakptr", llmemory.Address)) +WEAKREFPTR = lltype.Ptr(WEAKREF) +sizeof_weakref= llmemory.sizeof(WEAKREF) +empty_weakref = lltype.malloc(WEAKREF, immortal=True) +empty_weakref.weakptr = llmemory.NULL + +def ll_weakref_deref(wref): + wref = llmemory.cast_weakrefptr_to_ptr(WEAKREFPTR, wref) + if wref: + return wref.weakptr + return llmemory.NULL + +def convert_weakref_to(targetptr): + # Prebuilt weakrefs don't really need to be weak at all, + # but we need to emulate the structure expected by ll_weakref_deref(). + if not targetptr: + return empty_weaklink + else: + link = lltype.malloc(WEAKPTR, immortal=True) + link.weakptr = llmemory.cast_ptr_to_adr(targetptr) + return link Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Wed Sep 12 22:15:04 2007 @@ -388,6 +388,49 @@ res = run([5, 42]) #XXX pure lazyness here too assert 160 <= res <= 165 + def test_weakref(self): + import weakref, gc + class A(object): + pass + def g(): + a = A() + return weakref.ref(a) + def f(): + a = A() + ref = weakref.ref(a) + result = ref() is a + ref = g() + llop.gc__collect(lltype.Void) + result = result and (ref() is None) + # check that a further collection is fine + llop.gc__collect(lltype.Void) + result = result and (ref() is None) + return result + run = self.runner(f) + res = run([]) + assert res + + def test_weakref_to_object_with_finalizer(self): + import weakref, gc + class A(object): + count = 0 + a = A() + class B(object): + def __del__(self): + a.count += 1 + def g(): + b = B() + return weakref.ref(b) + def f(): + ref = g() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + result = a.count == 1 and (ref() is None) + return result + run = self.runner(f) + res = run([]) + assert res + def test_collect_during_collect(self): class B(object): pass From cfbolz at codespeak.net Thu Sep 13 00:23:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Sep 2007 00:23:52 +0200 (CEST) Subject: [pypy-svn] r46521 - pypy/dist/pypy/translator/c/test Message-ID: <20070912222352.501D380F0@code0.codespeak.net> Author: cfbolz Date: Thu Sep 13 00:23:51 2007 New Revision: 46521 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: kill more tests belonging to the old way to do weakrefs Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Sep 13 00:23:51 2007 @@ -566,75 +566,6 @@ # does not crash fn() - def test_framework_weakref_alive(self): - def func(): - f = Weakrefable() - f.x = 32 - ref1 = get_weakref(f) - ref2 = get_weakref(f) - return f.x + ref2.ref().x - assert func() == 64 - f = self.getcompiled(func) - assert f() == 64 - - def test_framework_weakref_dying(self): - def g(): - f = Weakrefable() - f.x = 32 - return get_weakref(f) - def func(): - ref = g() - llop.gc__collect(lltype.Void) - llop.gc__collect(lltype.Void) - return ref.ref() is None - f = self.getcompiled(func) - assert f() - - def test_framework_weakref_callback(self): - global_w = Weakrefable() - global_w.x = 31 - def g(ref): - global_w.x = 32 - def h(): - f = Weakrefable() - f.x = 32 - ref = get_weakref(f, g) - return ref - def func(): - ref = h() - llop.gc__collect(lltype.Void) - llop.gc__collect(lltype.Void) - return (ref.ref() is None and ref.callback is not None and - global_w.x == 32) - f = self.getcompiled(func) - assert f() - - def test_framework_weakref_dont_always_callback(self): - global_w1 = Weakrefable() - global_w1.x = 30 - global_w2 = Weakrefable() - global_w2.x = 30 - def g1(ref): - global_w1.x = 32 - def g2(ref): - global_w2.x = 32 - def h(): - f = Weakrefable() - f.x = 32 - ref1 = get_weakref(f, g1) - ref2 = get_weakref(f, g2) - if f.x % 2 == 0: # start a new block to make sure ref2 dies before f - ref2 = None - f.x = 12 - llop.gc__collect(lltype.Void) - llop.gc__collect(lltype.Void) - return ref1 - def func(): - ref = h() - return global_w1.x == 32 and global_w2.x == 30 - f = compile_func(func, []) - assert f() - def test_framework_malloc_raw(self): A = lltype.Struct('A', ('value', lltype.Signed)) From cfbolz at codespeak.net Thu Sep 13 00:39:26 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Sep 2007 00:39:26 +0200 (CEST) Subject: [pypy-svn] r46524 - in pypy/dist/pypy/translator/c: . test Message-ID: <20070912223926.7C43780C3@code0.codespeak.net> Author: cfbolz Date: Thu Sep 13 00:39:26 2007 New Revision: 46524 Modified: pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: make weakrefs work with genc and mark-n-sweep too. gives an assertion error when using the framework GC for some reason :-( Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Thu Sep 13 00:39:26 2007 @@ -358,6 +358,9 @@ fnptr = self.db.gctransformer.frameworkgc_setup_ptr.value yield '%s();' % (self.db.get(fnptr),) + def get_real_weakref_type(self): + return framework.WEAKREF + def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] # XXX this more or less assumes mark-and-sweep gc Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Sep 13 00:39:26 2007 @@ -566,6 +566,41 @@ # does not crash fn() + def test_weakref(self): + import weakref + from pypy.rlib import rgc + + class A: + pass + + def fn(): + n = 7000 + keepalive = [] + weakrefs = [] + a = None + for i in range(n): + if i & 1 == 0: + a = A() + a.index = i + weakrefs.append(weakref.ref(a)) + if i % 7 == 6: + keepalive.append(a) + rgc.collect() + count_free = 0 + for i in range(n): + a = weakrefs[i]() + if i % 7 == 6: + assert a is not None + if a is not None: + assert a.index == i & ~1 + else: + count_free += 1 + return count_free + c_fn = self.getcompiled(fn) + res = c_fn() + # more than half of them should have been freed, ideally up to 6000 + assert 3500 <= res <= 6000 + def test_framework_malloc_raw(self): A = lltype.Struct('A', ('value', lltype.Signed)) @@ -722,3 +757,6 @@ else: return res return compiled + + def test_weakref(self): + py.test.skip("fails for some reason I couldn't figure out yet :-(") From cfbolz at codespeak.net Thu Sep 13 01:23:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Sep 2007 01:23:15 +0200 (CEST) Subject: [pypy-svn] r46528 - in pypy/dist/pypy: rpython/memory/gctransform translator/c translator/c/test Message-ID: <20070912232315.A461680A7@code0.codespeak.net> Author: cfbolz Date: Thu Sep 13 01:23:15 2007 New Revision: 46528 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: whack. support prebuilt weakrefs with mark-n-sweep too Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Thu Sep 13 01:23:15 2007 @@ -727,8 +727,8 @@ # Prebuilt weakrefs don't really need to be weak at all, # but we need to emulate the structure expected by ll_weakref_deref(). if not targetptr: - return empty_weaklink + return empty_weakref else: - link = lltype.malloc(WEAKPTR, immortal=True) + link = lltype.malloc(WEAKREF, immortal=True) link.weakptr = llmemory.cast_ptr_to_adr(targetptr) return link Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Thu Sep 13 01:23:15 2007 @@ -361,6 +361,13 @@ def get_real_weakref_type(self): return framework.WEAKREF + def convert_weakref_to(self, ptarget): + result = framework.convert_weakref_to(ptarget) + # XXX whack + gct = self.db.gctransformer + gct.gcdata.gc.gcheaderbuilder.new_header(result) + return result + def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] # XXX this more or less assumes mark-and-sweep gc Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Sep 13 01:23:15 2007 @@ -601,6 +601,28 @@ # more than half of them should have been freed, ideally up to 6000 assert 3500 <= res <= 6000 + def test_prebuilt_weakref(self): + import weakref + from pypy.rlib import rgc + class A: + pass + a = A() + a.hello = 42 + refs = [weakref.ref(a), weakref.ref(A())] + rgc.collect() + def fn(): + result = 0 + for i in range(2): + a = refs[i]() + if a is None: + result += i + else: + result += a.hello * i + return result + c_fn = self.getcompiled(fn) + res = c_fn() + assert res == fn() + def test_framework_malloc_raw(self): A = lltype.Struct('A', ('value', lltype.Signed)) From arigo at codespeak.net Thu Sep 13 10:43:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Sep 2007 10:43:54 +0200 (CEST) Subject: [pypy-svn] r46529 - in pypy/dist/pypy: interpreter module/_weakref translator/goal Message-ID: <20070913084354.3193880C3@code0.codespeak.net> Author: arigo Date: Thu Sep 13 10:43:53 2007 New Revision: 46529 Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/mixedmodule.py pypy/dist/pypy/module/_weakref/__init__.py pypy/dist/pypy/translator/goal/targetpypystandalone.py Log: Revert the SkipModule logic introduced in r46443. Instead, let's use the same style as already present in targetpypystandalone, which has the advantage of detecting inconsistent option combinations like --gc=ref --withmod-_weakref. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Thu Sep 13 10:43:53 2007 @@ -246,7 +246,6 @@ def setbuiltinmodule(self, importname): """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules""" import sys - from pypy.interpreter.mixedmodule import SkipModule fullname = "pypy.module.%s" % importname @@ -258,10 +257,7 @@ name = importname w_name = self.wrap(name) - try: - w_mod = self.wrap(Module(self, w_name)) - except SkipModule: - return None + w_mod = self.wrap(Module(self, w_name)) w_modules = self.sys.get('modules') self.setitem(w_modules, w_name, w_mod) return name Modified: pypy/dist/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/dist/pypy/interpreter/mixedmodule.py (original) +++ pypy/dist/pypy/interpreter/mixedmodule.py Thu Sep 13 10:43:53 2007 @@ -7,9 +7,6 @@ import inspect -class SkipModule(Exception): - """Skip the installation of this built-in module.""" - class MixedModule(Module): NOT_RPYTHON_ATTRIBUTES = ['loaders'] Modified: pypy/dist/pypy/module/_weakref/__init__.py ============================================================================== --- pypy/dist/pypy/module/_weakref/__init__.py (original) +++ pypy/dist/pypy/module/_weakref/__init__.py Thu Sep 13 10:43:53 2007 @@ -1,4 +1,4 @@ -from pypy.interpreter.mixedmodule import MixedModule, SkipModule +from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): appleveldefs = { @@ -12,8 +12,3 @@ 'CallableProxyType': 'interp__weakref.W_CallableProxy', 'proxy': 'interp__weakref.proxy' } - - def __init__(self, space, *args): - if not space.config.translation.rweakref: - raise SkipModule("no RPython-level weakrefs") - MixedModule.__init__(self, space, *args) Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Thu Sep 13 10:43:53 2007 @@ -134,6 +134,9 @@ elif config.objspace.usemodules._stackless: config.translation.stackless = True + if not config.translation.rweakref: + config.objspace.usemodules._weakref = False + if self.translateconfig.goal_options.jit: config.objspace.usemodules.pypyjit = True elif config.objspace.usemodules.pypyjit: From arigo at codespeak.net Thu Sep 13 11:09:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Sep 2007 11:09:21 +0200 (CEST) Subject: [pypy-svn] r46530 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20070913090921.8437380CE@code0.codespeak.net> Author: arigo Date: Thu Sep 13 11:09:20 2007 New Revision: 46530 Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py Log: Forgot this. Hard to test though. Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Thu Sep 13 11:09:20 2007 @@ -160,6 +160,8 @@ def ll_weakref_create(targetaddr): link = llop.boehm_malloc_atomic(llmemory.Address, sizeof_weakreflink) + if not link: + raise MemoryError plink = llmemory.cast_adr_to_ptr(link, lltype.Ptr(WEAKLINK)) plink[0] = targetaddr llop.boehm_disappearing_link(lltype.Void, link, targetaddr) From antocuni at codespeak.net Thu Sep 13 11:11:38 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 13 Sep 2007 11:11:38 +0200 (CEST) Subject: [pypy-svn] r46531 - in pypy/dist/pypy/translator: cli jvm llvm Message-ID: <20070913091138.2012A80C3@code0.codespeak.net> Author: antocuni Date: Thu Sep 13 11:11:37 2007 New Revision: 46531 Modified: pypy/dist/pypy/translator/cli/cts.py pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/llvm/database.py Log: comment out references to WeakGcAddress; should these be replaced with something else? Modified: pypy/dist/pypy/translator/cli/cts.py ============================================================================== --- pypy/dist/pypy/translator/cli/cts.py (original) +++ pypy/dist/pypy/translator/cli/cts.py Thu Sep 13 11:11:37 2007 @@ -7,7 +7,6 @@ from py.builtin import set from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.rpython.lltypesystem.llmemory import WeakGcAddress from pypy.translator.cli.option import getoption from pypy.translator.cli import oopspec @@ -133,7 +132,7 @@ ootype.Class: types.type, ootype.String: types.string, ootype.StringBuilder: types.string_builder, - WeakGcAddress: types.weakref, + #WeakGcAddress: types.weakref, # maps generic types to their ordinal ootype.List.SELFTYPE_T: types.list, Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Thu Sep 13 11:11:37 2007 @@ -17,7 +17,6 @@ jStringBuilder, jInt, jVoid, jString, jChar, jObject, \ jThrowable, JvmNativeClass from pypy.translator.jvm.builtin import JvmBuiltInType -from pypy.rpython.lltypesystem.llmemory import WeakGcAddress from pypy.translator.oosupport.database import Database as OODatabase from pypy.rpython.ootypesystem.bltregistry import ExternalType @@ -461,7 +460,7 @@ ootype.UniChar: jvmtype.jChar, ootype.Class: jvmtype.jClass, ootype.ROOT: jvmtype.jObject, # treat like a scalar - WeakGcAddress: jvmtype.jWeakRef, + #WeakGcAddress: jvmtype.jWeakRef, } # Dictionary for non-scalar types; in this case, if we see the key, we Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Sep 13 11:11:37 2007 @@ -193,8 +193,8 @@ ct = lltype.typeOf(ptrvalue) else: return - elif ct is llmemory.WeakGcAddress: - return # XXX sometime soon +## elif ct is llmemory.WeakGcAddress: +## return # XXX sometime soon else: return else: @@ -368,7 +368,7 @@ lltype.UnsignedLongLong: "ulong", lltype.SignedLongLong: "long", llmemory.Address: "sbyte*", - llmemory.WeakGcAddress: "sbyte*", + #llmemory.WeakGcAddress: "sbyte*", } # 32 bit platform @@ -396,7 +396,7 @@ lltype.Bool : self.repr_bool, lltype.Void : self.repr_void, llmemory.Address : self.repr_address, - llmemory.WeakGcAddress : self.repr_weakgcaddress, + #llmemory.WeakGcAddress : self.repr_weakgcaddress, } try: From cfbolz at codespeak.net Thu Sep 13 11:33:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Sep 2007 11:33:10 +0200 (CEST) Subject: [pypy-svn] r46533 - pypy/dist/pypy/rpython/memory Message-ID: <20070913093310.6941F80CD@code0.codespeak.net> Author: cfbolz Date: Thu Sep 13 11:33:10 2007 New Revision: 46533 Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py Log: shallowly fix the refcounting tests. They don't support weakrefs yet, though. Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Thu Sep 13 11:33:10 2007 @@ -74,10 +74,13 @@ return _varsize_offset_to_length[typeid] def varsize_offsets_to_gcpointers_in_var_part(typeid): return _varsize_offsets_to_gcpointers_in_var_part[typeid] + def weakpointer_offset(typeid): + return -1 return (is_varsize, getfinalizer, offsets_to_gc_pointers, fixed_size, varsize_item_sizes, varsize_offset_to_variable_part, varsize_offset_to_length, - varsize_offsets_to_gcpointers_in_var_part) + varsize_offsets_to_gcpointers_in_var_part, + weakpointer_offset) def is_varsize(self, typeid): assert typeid >= 0 @@ -126,12 +129,16 @@ else: return 0 + def weakpointer_offset(self, typeid): + return -1 + def get_setup_query_functions(self): return (self.is_varsize, self.getfinalizer, self.offsets_to_gc_pointers, self.fixed_size, self.varsize_item_sizes, self.varsize_offset_to_variable_part, self.varsize_offset_to_length, - self.varsize_offsets_to_gcpointers_in_var_part) + self.varsize_offsets_to_gcpointers_in_var_part, + self.weakpointer_offset) def getfunctionptr(annotator, graphfunc): @@ -246,10 +253,10 @@ AddressLinkedList = self.AddressLinkedList def instantiate_linked_list(): return AddressLinkedList() - f1, f2, f3, f4, f5, f6, f7, f8 = self.query_types.create_query_functions() + f1, f2, f3, f4, f5, f6, f7, f8, f9 = self.query_types.create_query_functions() the_gc = gc_class(AddressLinkedList) def instantiate_gc(): - the_gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7, f8) + the_gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7, f8, f9) the_gc.setup() return the_gc func, dummy_get_roots1, dummy_get_roots2 = gc.get_dummy_annotate( From arigo at codespeak.net Thu Sep 13 17:27:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Sep 2007 17:27:09 +0200 (CEST) Subject: [pypy-svn] r46538 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20070913152709.67F0980F4@code0.codespeak.net> Author: arigo Date: Thu Sep 13 17:27:08 2007 New Revision: 46538 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py Log: Folding these doesn't work currently anyway. Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Thu Sep 13 17:27:08 2007 @@ -406,8 +406,8 @@ # in the framework GC, so it should never cause a stack unwind. 'weakref_create': LLOp(canraise=(MemoryError,), sideeffects=False), 'weakref_deref': LLOp(sideeffects=False), - 'cast_ptr_to_weakrefptr': LLOp(canfold=True), # no-op, pure type hiding - 'cast_weakrefptr_to_ptr': LLOp(canfold=True), # no-op, pure type revealing + 'cast_ptr_to_weakrefptr': LLOp(sideeffects=False), # no-op type hiding + 'cast_weakrefptr_to_ptr': LLOp(sideeffects=False), # no-op type revealing # __________ stackless operation(s) __________ From cfbolz at codespeak.net Thu Sep 13 17:28:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Sep 2007 17:28:25 +0200 (CEST) Subject: [pypy-svn] r46539 - pypy/dist/pypy/translator/tool Message-ID: <20070913152825.C252580F4@code0.codespeak.net> Author: cfbolz Date: Thu Sep 13 17:28:25 2007 New Revision: 46539 Modified: pypy/dist/pypy/translator/tool/lltracker.py Log: make the ref tracker support addresses again Modified: pypy/dist/pypy/translator/tool/lltracker.py ============================================================================== --- pypy/dist/pypy/translator/tool/lltracker.py (original) +++ pypy/dist/pypy/translator/tool/lltracker.py Thu Sep 13 17:28:25 2007 @@ -100,16 +100,16 @@ if not o: yield name, 'NULL' else: - addrof = o.get() + addrof = o.ref() T1 = lltype.typeOf(addrof) if (isinstance(T1, lltype.Ptr) and isinstance(T1.TO, lltype.Struct) and addrof._obj in header2obj): yield name + ' @hdr', self.normalize(addrof._obj) else: - yield name + ' @', self.normalize(o.ob._obj) - if o.offset: - yield '... offset', str(o.offset) + yield name + ' @', self.normalize(o.ptr._obj) +## if o.offset: +## yield '... offset', str(o.offset) else: yield name, str(o) From simonb at codespeak.net Thu Sep 13 18:21:32 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Thu, 13 Sep 2007 18:21:32 +0200 (CEST) Subject: [pypy-svn] r46540 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070913162132.219E280DA@code0.codespeak.net> Author: simonb Date: Thu Sep 13 18:21:29 2007 New Revision: 46540 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: get inplace operators working, also plays nice with coercion rules Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Thu Sep 13 18:21:29 2007 @@ -88,48 +88,63 @@ return SomeArray(typecode, self.ndim) raise AnnotatorError() -class __extend__(pairtype(SomeArray, SomeArray)): +def unify_scalar_tp(item1, item2): + typecode = None + if float in (item1.knowntype, item2.knowntype): + typecode = 'd' + else: + item_knowntype = rarithmetic.compute_restype(item1.knowntype, item2.knowntype) + for typecode, s_item in SomeArray.typecode_to_item.items(): + if s_item.knowntype == item_knowntype: + break + if typecode is None: + raise AnnotatorError() + return typecode - def add((s_arr1, s_arr2)): - item1 = s_arr1.get_item_type() - item2 = s_arr2.get_item_type() - typecode = None - if float in (item1.knowntype, item2.knowntype): - typecode = 'd' +class __extend__(pairtype(SomeArray, SomeObject)): + def add((s_arr, s_other)): + item1 = s_arr.get_item_type() + ndim = s_arr.ndim + if isinstance(s_other, SomeArray): + item2 = s_other.get_item_type() + ndim = max(ndim, s_other.ndim) + elif isinstance(s_other, SomeList): + item2 = s_other.listdef.listitem.s_value + elif isinstance(s_other, SomeFloat): + item2 = s_other else: - item_knowntype = rarithmetic.compute_restype(item1.knowntype, item2.knowntype) - for typecode, s_item in SomeArray.typecode_to_item.items(): - if s_item.knowntype == item_knowntype: - break - if typecode is None: - raise AnnotatorError() - ndim = max(s_arr1.ndim, s_arr2.ndim) - return SomeArray(typecode, s_arr1.ndim) - - # union ? + raise AnnotatorError("cannot operate with %s"%s_other) + typecode = unify_scalar_tp(item1, item2) + return SomeArray(typecode, ndim) sub = mul = div = add -class __extend__(pairtype(SomeArray, SomeFloat)): - def add((s_arr, s_flt)): - item = s_arr.get_item_type() - typecode = None - if float in (item.knowntype, s_flt.knowntype): - typecode = 'd' - else: - item_knowntype = rarithmetic.compute_restype(item.knowntype, s_flt.knowntype) - for typecode, s_item in SomeArray.typecode_to_item.items(): - if s_item.knowntype == item_knowntype: - break - if typecode is None: - raise AnnotatorError() - return SomeArray(typecode, s_arr.ndim) - # union ? - sub = mul = div = add + def inplace_add((s_arr, s_other)): + if isinstance(s_other, SomeArray): + # This can only work if s_other is broadcastable to s_arr + if s_other.ndim > s_arr.ndim: + raise AnnotatorError("invalid return array shape") + elif isinstance(s_other, SomeList): + item2 = s_other.listdef.listitem.s_value + if not isinstance(item2, SomeFloat): + raise AnnotatorError("cannot operate with list of %s"%item2) + elif not isinstance(s_other, SomeFloat): + raise AnnotatorError("cannot operate with %s"%s_other) + return s_arr + inplace_sub = inplace_mul = inplace_div = inplace_add class __extend__(pairtype(SomeFloat, SomeArray)): def add((s_flt, s_arr)): return pair(s_arr, s_flt).add() - # union ? + sub = mul = div = add + + def inplace_add((s_flt, s_arr)): + # This involves promoting the type of the lhs + raise AnnotatorError() + inplace_sub = inplace_mul = inplace_div = inplace_add + +class __extend__(pairtype(SomeList, SomeArray)): + def add((s_lst, s_arr)): + return pair(s_arr, s_lst).add() sub = mul = div = add class __extend__(pairtype(SomeArray, SomeTuple)): @@ -183,6 +198,12 @@ s_tuple = SomeTuple([s_index]) return pair(s_array, s_tuple).getitem() +def build_annotation_from_scalar(s_value): + key = type(s_value), s_value.knowntype + typecode = numpy_typedict[key] + ndim = 1 + return SomeArray(typecode, ndim) + class ArrayCallEntry(ExtRegistryEntry): _about_ = numpy.array Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Thu Sep 13 18:21:29 2007 @@ -311,7 +311,6 @@ hop.gendirectcall(ll_array_set, cITEM, v_it0, v_it1) return v_result - def get_ndim(self, hop, v_array): cname = inputconst(Void, 'ndim') return hop.llops.genop('getfield', [v_array, cname], resulttype=Signed) @@ -343,10 +342,14 @@ key = self.__class__, self.typecode, self.ndim return key +#______________________________________________________________________________ + def ll_array_binop(it0, it1, it2, binop): debug_assert(it0.size == it1.size, "it0.size == it1.size") debug_assert(it1.size == it2.size, "it0.size == it1.size") while it0.index < it0.size: + # We don't need a cast here, because it0.dataptr[0] is always + # big enough to contain the result. it0.dataptr[0] = binop(it1.dataptr[0], it2.dataptr[0]) it0.ll_next() it1.ll_next() @@ -358,7 +361,6 @@ # from the largest of the two args: v_array0 = hop.gendirectcall(ll_build_like2, cARRAY, v_array1, v_array2) iter_new, iter_broadcast = gen_iter_funcs(r_array0.ndim) - iter_new._annenforceargs_ = [None, None, None, None] cITER0 = hop.inputconst(Void, r_array0.ITER.TO) cITER1 = hop.inputconst(Void, r_array0.build_ITER_for(r_array1)) cITER2 = hop.inputconst(Void, r_array0.build_ITER_for(r_array2)) @@ -375,15 +377,18 @@ v_array1, v_array2 = hop.inputargs(r_array1, r_array2) if isinstance(r_array1.lowleveltype, Primitive): - r_array1, v_array1 = convert_scalar_to_array(r_array0, v_array1, hop.llops) + r_array1, v_array1 = convert_scalar_to_array(r_array1, v_array1, hop.llops) + elif isinstance(r_array1, AbstractBaseListRepr): + r_array1, v_array1 = convert_list_to_array(r_array1, v_array1, hop.llops) elif not isinstance(r_array1, ArrayRepr): raise TyperError("can't operate with %s"%r_array1) if isinstance(r_array2.lowleveltype, Primitive): - r_array2, v_array2 = convert_scalar_to_array(r_array0, v_array2, hop.llops) + r_array2, v_array2 = convert_scalar_to_array(r_array2, v_array2, hop.llops) + elif isinstance(r_array2, AbstractBaseListRepr): + r_array2, v_array2 = convert_list_to_array(r_array2, v_array2, hop.llops) elif not isinstance(r_array2, ArrayRepr): raise TyperError("can't operate with %s"%r_array2) - print "rtype_binop", binop, binop(2,3) return _rtype_binop(r_array0, r_array1, r_array2, v_array1, v_array2, hop, binop) for tp in (pairtype(ArrayRepr, ArrayRepr), @@ -395,6 +400,55 @@ (lambda a,b:a/b, "rtype_div")): setattr(tp, methname, lambda self, hop, binop=binop : rtype_binop(self, hop, binop)) +#______________________________________________________________________________ + +def ll_array_inplace_binop(ITEM, it0, it1, binop): + debug_assert(it0.size == it1.size, "it0.size == it1.size") + while it0.index < it0.size: + it0.dataptr[0] = cast_primitive(ITEM, binop(it0.dataptr[0], it1.dataptr[0])) + it0.ll_next() + it1.ll_next() + +def _rtype_inplace_binop(r_array0, r_array1, r_array2, v_array1, v_array2, hop, binop): + iter_new, iter_broadcast = gen_iter_funcs(r_array1.ndim) + cITEM = hop.inputconst(Void, r_array1.ITEM) + cITER1 = hop.inputconst(Void, r_array1.ITER.TO) + cITER2 = hop.inputconst(Void, r_array1.build_ITER_for(r_array2)) + cbroadcast = hop.inputconst(Void, iter_broadcast) + v_it1 = hop.gendirectcall(iter_new, cITER1, v_array1, v_array1, cbroadcast) + v_it2 = hop.gendirectcall(iter_new, cITER2, v_array2, v_array1, cbroadcast) + cbinop = hop.inputconst(Void, binop) + hop.gendirectcall(ll_array_inplace_binop, cITEM, v_it1, v_it2, cbinop) + return v_array1 + +def rtype_inplace_binop((r_array1, r_array2), hop, binop): + r_array0 = hop.r_result + v_array1, v_array2 = hop.inputargs(r_array1, r_array2) + + if isinstance(r_array1.lowleveltype, Primitive): + raise TyperError("can't operate with %s"%r_array1) + elif not isinstance(r_array1, ArrayRepr): + raise TyperError("can't operate with %s"%r_array1) + + if isinstance(r_array2.lowleveltype, Primitive): + r_array2, v_array2 = convert_scalar_to_array(r_array2, v_array2, hop.llops) + elif isinstance(r_array2, AbstractBaseListRepr): + r_array2, v_array2 = convert_list_to_array(r_array2, v_array2, hop.llops) + elif not isinstance(r_array2, ArrayRepr): + raise TyperError("can't operate with %s"%r_array2) + return _rtype_inplace_binop(r_array0, r_array1, r_array2, v_array1, v_array2, hop, binop) + +for tp in (pairtype(ArrayRepr, ArrayRepr), + pairtype(ArrayRepr, Repr), + pairtype(Repr, ArrayRepr)): + for (binop, methname) in ((lambda a,b:a+b, "rtype_inplace_add"), + (lambda a,b:a-b, "rtype_inplace_sub"), + (lambda a,b:a*b, "rtype_inplace_mul"), + (lambda a,b:a/b, "rtype_inplace_div")): + setattr(tp, methname, lambda self, hop, binop=binop : rtype_inplace_binop(self, hop, binop)) + +#______________________________________________________________________________ + def gen_getset_item(ndim): unrolling_dims = unrolling_iterable(range(ndim)) def ll_get_item(ARRAY, ao, tpl): @@ -498,9 +552,10 @@ #v_array = llops.gendirectcall(ll_build_alias_to_list, cARRAY, v_list) # nice idea... return r_array, v_array -def convert_scalar_to_array(r_array, v_item, llops): +def convert_scalar_to_array(r_item, v_item, llops): # x -> array([x]) - s_array = r_array.s_array.get_one_dim() + s_item = lltype_to_annotation(r_item.lowleveltype) + s_array = aarray.build_annotation_from_scalar(s_item) r_array = llops.rtyper.getrepr(s_array) from pypy.rpython.rmodel import inputconst cARRAY = inputconst(Void, r_array.ARRAY.TO) @@ -568,7 +623,7 @@ source_ndim = r_item.ndim elif isinstance(r_item.lowleveltype, Primitive): # "broadcast" a scalar - r_item, v_item = convert_scalar_to_array(r_view, v_item, hop.llops) + r_item, v_item = convert_scalar_to_array(r_item, v_item, hop.llops) source_ndim = 1 elif isinstance(r_item, AbstractBaseListRepr): r_item, v_item = convert_list_to_array(r_item, v_item, hop.llops) Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Thu Sep 13 18:21:29 2007 @@ -116,6 +116,17 @@ s = a.build_types(f, []) assert s.typecode == 'i' + def test_annotate_array_add_list(self): + def f(): + a1 = numpy.array([1,2]) + a2 = [3., 4.] + return a1 + a2 + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert s.typecode == 'd' + def test_annotate_array_add_coerce(self): def f(): a1 = numpy.array([1,2]) @@ -127,7 +138,7 @@ s = a.build_types(f, []) assert s.typecode == 'd' - def test_annotate_array_add_2(self): + def test_annotate_array_add_scalar(self): def f(): a = numpy.array([1,2]) a = a + 3 @@ -138,6 +149,38 @@ s = a.build_types(f, []) assert s.typecode == 'i' + def test_annotate_array_add_scalar_coerce(self): + def f(): + a = numpy.array([1,2]) + return a + 3. + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert s.typecode == 'd' + + def test_annotate_array_inplace_add_list(self): + def f(): + a = numpy.array([1,2,3,4]) + a += [4,3,2,1] + return a + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert s.typecode == 'l' + + def test_annotate_array_inplace_mul_coerce(self): + def f(): + a = numpy.array([1,2,3,4]) + a *= 0.5 + return a + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert s.typecode == 'l' + def test_annotate_array_dtype(self): def f(): a1 = numpy.array([1,2], dtype='d') @@ -599,12 +642,31 @@ def test_specialize_array_add_1_1_coerce(self): def f(): a1 = numpy.array([1,2]) - a2 = numpy.array([6.,9.]) + a2 = numpy.array([6.5,9.5]) return a1 + a2 res = interpret(f, []) - assert res.data[0] == 7. - assert res.data[1] == 11. + assert res.data[0] == 7.5 + assert res.data[1] == 11.5 + + def test_specialize_array_add_1_1_coerce_from_list(self): + def f(): + a1 = numpy.array([1,2]) + a2 = [6.5,9.5] + return a1 + a2 + + res = interpret(f, []) + assert res.data[0] == 7.5 + assert res.data[1] == 11.5 + + def test_specialize_array_add_1_1_coerce_from_scalar(self): + def f(): + a = numpy.array([1,2]) + return a + 1.5 + + res = interpret(f, []) + assert res.data[0] == 2.5 + assert res.data[1] == 3.5 def test_specialize_array_add_2_1(self): def f(): @@ -631,6 +693,82 @@ for i in range(len(data)): assert res.dataptr[i] == 5*data[i] + def test_specialize_array_add_list(self): + def f(): + a = numpy.array([1,2,3,4]) + a = a + [4,3,2,1] + return a + + res = interpret(f, []) + for i in range(4): + assert res.dataptr[i] == 5 + + def test_specialize_array_radd_list(self): + def f(): + a = numpy.array([1,2,3,4]) + a = [4,3,2,1] + a + return a + + res = interpret(f, []) + for i in range(4): + assert res.dataptr[i] == 5 + + def test_specialize_array_inplace_add(self): + def f(): + a = numpy.array([1,2,3,4]) + a += 1 + return a + + res = interpret(f, []) + data = [1,2,3,4] + for i in range(len(data)): + assert res.dataptr[i] == data[i] + 1 + + def test_specialize_array_inplace_add_list_broadcast(self): + def f(): + a = numpy.array([1,2,3,4]).reshape((2,2)) + a += [0,1] + return a + + res = interpret(f, []) + data = [1,3,3,5] + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def test_specialize_array_inplace_mul_coerce(self): + def f(): + a = numpy.array([1,2,3,4]) + a *= numpy.array([0.5]) + return a + + res = interpret(f, []) + data = [1,2,3,4] + for i in range(len(data)): + assert res.dataptr[i] == data[i]//2 + + def test_specialize_array_inplace_mul_coerce_from_list(self): + def f(): + a = numpy.array([1,2]) + a *= [1.5, 2.5] + return a + + res = interpret(f, []) + data = [1.0, 5.0] + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def test_specialize_array_inplace_mul_coerce_from_scalar(self): + def f(): + a = numpy.array([1,2,3,4]) + a *= 0.5 + return a + + res = interpret(f, []) + data = [1,2,3,4] + for i in range(len(data)): + assert res.dataptr[i] == data[i]//2 + + class Test_compile: def setup_class(self): if not test_c_compile: From arigo at codespeak.net Thu Sep 13 19:01:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Sep 2007 19:01:00 +0200 (CEST) Subject: [pypy-svn] r46546 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20070913170100.22C5C80B0@code0.codespeak.net> Author: arigo Date: Thu Sep 13 19:00:59 2007 New Revision: 46546 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: - Start counting typeids at 1 instead of 0, with debug_asserts() to detect if somebody sees an uninitialized 0 typeid. This exposes the framework weakref bug... - Don't need to check for NULL weakrefptr in weakref_deref(). Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Thu Sep 13 19:00:59 2007 @@ -57,30 +57,39 @@ TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) def q_is_varsize(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].isvarsize def q_finalyzer(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].finalyzer def q_offsets_to_gc_pointers(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].ofstoptrs def q_fixed_size(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].fixedsize def q_varsize_item_sizes(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].varitemsize def q_varsize_offset_to_variable_part(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].ofstovar def q_varsize_offset_to_length(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].ofstolength def q_varsize_offsets_to_gcpointers_in_var_part(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].varofstoptrs def q_weakpointer_offset(typeid): + debug_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].weakptrofs gcdata = GCData() @@ -96,7 +105,9 @@ gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() self.gcdata = gcdata - self.type_info_list = [] + dummy = {"weakptrofs": -1, + "ofstolength": -1} + self.type_info_list = [dummy] # don't use typeid 0, helps debugging self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} self.static_gc_roots = [] @@ -719,9 +730,7 @@ def ll_weakref_deref(wref): wref = llmemory.cast_weakrefptr_to_ptr(WEAKREFPTR, wref) - if wref: - return wref.weakptr - return llmemory.NULL + return wref.weakptr def convert_weakref_to(targetptr): # Prebuilt weakrefs don't really need to be weak at all, From antocuni at codespeak.net Thu Sep 13 19:08:07 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 13 Sep 2007 19:08:07 +0200 (CEST) Subject: [pypy-svn] r46547 - in pypy/dist/pypy/rpython: . lltypesystem ootypesystem ootypesystem/test test Message-ID: <20070913170807.B246880B0@code0.codespeak.net> Author: antocuni Date: Thu Sep 13 19:08:07 2007 New Revision: 46547 Modified: pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py pypy/dist/pypy/rpython/ootypesystem/ootype.py pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rweakref.py pypy/dist/pypy/rpython/test/test_rweakref.py Log: implement rweakref for ootype. Modified: pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rbuiltin.py Thu Sep 13 19:08:07 2007 @@ -66,3 +66,40 @@ BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr BUILTIN_TYPER[__import__] = rtype_builtin___import__ BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict + +# _________________________________________________________________ +# weakrefs + +import weakref +from pypy.rpython.lltypesystem import llmemory + +def rtype_weakref_create(hop): + # Note: this code also works for the RPython-level calls 'weakref.ref(x)'. + vlist = hop.inputargs(hop.args_r[0]) + hop.exception_cannot_occur() + return hop.genop('weakref_create', vlist, resulttype=llmemory.WeakRefPtr) + +def rtype_weakref_deref(hop): + c_ptrtype, v_wref = hop.inputargs(lltype.Void, hop.args_r[1]) + assert v_wref.concretetype == llmemory.WeakRefPtr + hop.exception_cannot_occur() + return hop.genop('weakref_deref', [v_wref], resulttype=c_ptrtype.value) + +def rtype_cast_ptr_to_weakrefptr(hop): + vlist = hop.inputargs(hop.args_r[0]) + hop.exception_cannot_occur() + return hop.genop('cast_ptr_to_weakrefptr', vlist, + resulttype=llmemory.WeakRefPtr) + +def rtype_cast_weakrefptr_to_ptr(hop): + c_ptrtype, v_wref = hop.inputargs(lltype.Void, hop.args_r[1]) + assert v_wref.concretetype == llmemory.WeakRefPtr + hop.exception_cannot_occur() + return hop.genop('cast_weakrefptr_to_ptr', [v_wref], + resulttype=c_ptrtype.value) + +BUILTIN_TYPER[weakref.ref] = rtype_weakref_create +BUILTIN_TYPER[llmemory.weakref_create] = rtype_weakref_create +BUILTIN_TYPER[llmemory.weakref_deref ] = rtype_weakref_deref +BUILTIN_TYPER[llmemory.cast_ptr_to_weakrefptr] = rtype_cast_ptr_to_weakrefptr +BUILTIN_TYPER[llmemory.cast_weakrefptr_to_ptr] = rtype_cast_weakrefptr_to_ptr Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Thu Sep 13 19:08:07 2007 @@ -381,6 +381,25 @@ def _specialize(self, generic_types): return self +# WARNING: the name WeakReference is rebound at the end of file +class WeakReference(BuiltinADTType): + def __init__(self): + self._null = _null_weak_reference(self) + self._GENERIC_METHODS = frozendict({ + "ll_set": Meth([ROOT], Void), + "ll_deref": Meth([], ROOT), + }) + self._setup_methods({}) + + def _defl(self): + return self._null + + def _get_interp_class(self): + return _weak_reference + + def _specialize(self, generic_types): + return self + class List(BuiltinADTType): # placeholders for types # make sure that each derived class has his own SELFTYPE_T @@ -787,6 +806,9 @@ _enforce = _upcast def _downcast(self, INSTANCE): + if not self._inst: + assert isSubclass(INSTANCE, self._TYPE) or isSubclass(self._TYPE, INSTANCE) + return null(INSTANCE) assert isSubclass(INSTANCE, self._TYPE) return _view(INSTANCE, self._inst) @@ -1133,6 +1155,37 @@ def __init__(self, STRING_BUILDER): self.__dict__["_TYPE"] = STRING_BUILDER +import weakref + +class _weak_reference(_builtin_type): + def __init__(self, WEAK_REFERENCE): + self._TYPE = WEAK_REFERENCE + self._ref = None + + def _unwrap_view(self, obj): + # we can't store directly the view inside the weakref because + # the view can be a temp object that is not referenced + # anywhere else. + while isinstance(obj, _view): + obj = obj._inst + return obj + + def ll_set(self, target): + assert isinstance(typeOf(target), Instance) + target = self._unwrap_view(target) + self._ref = weakref.ref(target) + + def ll_deref(self): + if self._ref is None: + return null(ROOT) + result = self._ref() + if result is None: + return null(ROOT) + return _view(ROOT, result) + +class _null_weak_reference(_null_mixin(_weak_reference), _weak_reference): + def __init__(self, WEAK_REFERENCE): + self.__dict__["_TYPE"] = WEAK_REFERENCE class _list(_builtin_type): def __init__(self, LIST): @@ -1475,6 +1528,13 @@ def hasDictTypes(DICT): return DICT._is_initialized() +def ooweakref_create(obj): + ref = new(WeakReference) + ref.ll_set(obj) + return ref + +ROOT = Instance('Root', None, _is_root=True) String = String() StringBuilder = StringBuilder() -ROOT = Instance('Root', None, _is_root=True) +WeakReference = WeakReference() +dead_wref = new(WeakReference) Modified: pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py Thu Sep 13 19:08:07 2007 @@ -101,3 +101,18 @@ BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate + + +# _________________________________________________________________ +# weakrefs + +import weakref +from pypy.rpython.lltypesystem import llmemory + +def rtype_weakref_create(hop): + # Note: this code also works for the RPython-level calls 'weakref.ref(x)'. + vlist = hop.inputargs(hop.args_r[0]) + hop.exception_cannot_occur() + return hop.gendirectcall(ootype.ooweakref_create, *vlist) + +BUILTIN_TYPER[weakref.ref] = rtype_weakref_create Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py Thu Sep 13 19:08:07 2007 @@ -451,3 +451,27 @@ DT = DictItemsIterator(String, Signed) _, meth = DT._lookup('ll_go_next') assert meth._can_raise == True + +def test_cast_null(): + A = Instance("A", ROOT) + B = Instance("B", ROOT) + rootnull = null(ROOT) + anull = oodowncast(A, rootnull) + assert typeOf(anull) is A + assert ooupcast(ROOT, anull) == rootnull + py.test.raises(AssertionError, oodowncast, B, anull) + +def test_weak_reference(): + import gc + A = Instance("A", ROOT) + obj = new(A) + ref = new(WeakReference) + ref.ll_set(obj) + assert oodowncast(A, ref.ll_deref()) == obj + del obj + gc.collect() + assert ref.ll_deref() is null(ROOT) + +def test_dead_wref(): + ref = new(WeakReference) + assert ref.ll_deref() is null(ROOT) Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Thu Sep 13 19:08:07 2007 @@ -594,42 +594,6 @@ BUILTIN_TYPER[llmemory.offsetof] = rtype_offsetof # _________________________________________________________________ -# weakrefs - -import weakref - -def rtype_weakref_create(hop): - # Note: this code also works for the RPython-level calls 'weakref.ref(x)'. - vlist = hop.inputargs(hop.args_r[0]) - hop.exception_cannot_occur() - return hop.genop('weakref_create', vlist, resulttype=llmemory.WeakRefPtr) - -def rtype_weakref_deref(hop): - c_ptrtype, v_wref = hop.inputargs(lltype.Void, hop.args_r[1]) - assert v_wref.concretetype == llmemory.WeakRefPtr - hop.exception_cannot_occur() - return hop.genop('weakref_deref', [v_wref], resulttype=c_ptrtype.value) - -def rtype_cast_ptr_to_weakrefptr(hop): - vlist = hop.inputargs(hop.args_r[0]) - hop.exception_cannot_occur() - return hop.genop('cast_ptr_to_weakrefptr', vlist, - resulttype=llmemory.WeakRefPtr) - -def rtype_cast_weakrefptr_to_ptr(hop): - c_ptrtype, v_wref = hop.inputargs(lltype.Void, hop.args_r[1]) - assert v_wref.concretetype == llmemory.WeakRefPtr - hop.exception_cannot_occur() - return hop.genop('cast_weakrefptr_to_ptr', [v_wref], - resulttype=c_ptrtype.value) - -BUILTIN_TYPER[weakref.ref] = rtype_weakref_create -BUILTIN_TYPER[llmemory.weakref_create] = rtype_weakref_create -BUILTIN_TYPER[llmemory.weakref_deref ] = rtype_weakref_deref -BUILTIN_TYPER[llmemory.cast_ptr_to_weakrefptr] = rtype_cast_ptr_to_weakrefptr -BUILTIN_TYPER[llmemory.cast_weakrefptr_to_ptr] = rtype_cast_weakrefptr_to_ptr - -# _________________________________________________________________ # non-gc objects def rtype_free_non_gc_object(hop): Modified: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/rweakref.py (original) +++ pypy/dist/pypy/rpython/rweakref.py Thu Sep 13 19:08:07 2007 @@ -5,6 +5,7 @@ from pypy.rpython.rmodel import Repr from pypy.rpython.rclass import getinstancerepr from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.ootypesystem import ootype # ____________________________________________________________ # @@ -12,13 +13,14 @@ class __extend__(annmodel.SomeWeakRef): def rtyper_makerepr(self, rtyper): - return WeakRefRepr(rtyper) + if rtyper.type_system.name == 'lltypesystem': + return LLWeakRefRepr(rtyper) + else: + return OOWeakRefRepr(rtyper) def rtyper_makekey(self): return self.__class__, - -class WeakRefRepr(Repr): - lowleveltype = llmemory.WeakRefPtr +class BaseWeakRefRepr(Repr): def __init__(self, rtyper): self.rtyper = rtyper @@ -26,11 +28,6 @@ raise TyperError("RPython-level weakrefs are not supported by " "this backend or GC policy") - def rtype_simple_call(self, hop): - v_wref, = hop.inputargs(self) - hop.exception_cannot_occur() - return hop.genop('weakref_deref', [v_wref], resulttype=hop.r_result) - def convert_const(self, value): assert isinstance(value, weakref.ReferenceType) instance = value() @@ -38,8 +35,37 @@ # obscure! if the annotator hasn't seen this object before, # we don't want to look at it now (confusion tends to result). if instance is None or not bk.have_seen(instance): - return llmemory.dead_wref + return self.dead_wref else: repr = self.rtyper.bindingrepr(Constant(instance)) llinstance = repr.convert_const(instance) - return llmemory.weakref_create(llinstance) + return self._weakref_create(llinstance) + + def _weakref_create(self, llinstance): + raise NotImplementedError + +class LLWeakRefRepr(BaseWeakRefRepr): + lowleveltype = llmemory.WeakRefPtr + dead_wref = llmemory.dead_wref + + def rtype_simple_call(self, hop): + v_wref, = hop.inputargs(self) + hop.exception_cannot_occur() + return hop.genop('weakref_deref', [v_wref], resulttype=hop.r_result) + + def _weakref_create(self, llinstance): + return llmemory.weakref_create(llinstance) + +class OOWeakRefRepr(BaseWeakRefRepr): + lowleveltype = ootype.WeakReference + dead_wref = ootype.dead_wref + + def rtype_simple_call(self, hop): + v_wref, = hop.inputargs(self) + cname = hop.inputconst(ootype.Void, 'll_deref') + hop.exception_cannot_occur() + v_deref = hop.genop('oosend', [cname, v_wref], resulttype=ootype.ROOT) + return hop.genop('oodowncast', [v_deref], resulttype=hop.r_result) + + def _weakref_create(self, llinstance): + return ootype.ooweakref_create(llinstance) Modified: pypy/dist/pypy/rpython/test/test_rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rweakref.py (original) +++ pypy/dist/pypy/rpython/test/test_rweakref.py Thu Sep 13 19:08:07 2007 @@ -1,89 +1,95 @@ import py, weakref from pypy.rlib import rgc from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +class BaseTestRweakref(BaseRtypingTest): -def test_ll_weakref(): - S = lltype.GcStruct('S', ('x',lltype.Signed)) - def g(): - s = lltype.malloc(S) - w = llmemory.weakref_create(s) - assert llmemory.weakref_deref(lltype.Ptr(S), w) == s - assert llmemory.weakref_deref(lltype.Ptr(S), w) == s - return w # 's' is forgotten here - def f(): - w = g() + def test_weakref_simple(self): + class A: + pass + class B(A): + pass + class C(A): + pass + + def f(n): + if n: + x = B() + x.hello = 42 + r = weakref.ref(x) + else: + x = C() + x.hello = 64 + r = weakref.ref(x) + return r().hello, x # returns 'x' too, to keep it alive + res = self.interpret(f, [1]) + assert res.item0 == 42 + res = self.interpret(f, [0]) + assert res.item0 == 64 + + def test_prebuilt_weakref(self): + class A: + pass + a1 = A() + a1.hello = 5 + w1 = weakref.ref(a1) + a2 = A() + a2.hello = 8 + w2 = weakref.ref(a2) + + def f(n): + if n: + r = w1 + else: + r = w2 + return r().hello + res = self.interpret(f, [1]) + assert res == 5 + res = self.interpret(f, [0]) + assert res == 8 + + def test_prebuilt_dead_weakref(self): + class A: + pass + a1 = A() + w1 = weakref.ref(a1) + a2 = A() + w2 = weakref.ref(a2) + + del a1 rgc.collect() - return llmemory.weakref_deref(lltype.Ptr(S), w) + assert w1() is None + + def f(n): + if n: + r = w1 + else: + r = w2 + return r() is not None + res = self.interpret(f, [1]) + assert res == False + res = self.interpret(f, [0]) + assert res == True + + +class TestLLtype(BaseTestRweakref, LLRtypeMixin): + def test_ll_weakref(self): + S = lltype.GcStruct('S', ('x',lltype.Signed)) + def g(): + s = lltype.malloc(S) + w = llmemory.weakref_create(s) + assert llmemory.weakref_deref(lltype.Ptr(S), w) == s + assert llmemory.weakref_deref(lltype.Ptr(S), w) == s + return w # 's' is forgotten here + def f(): + w = g() + rgc.collect() + return llmemory.weakref_deref(lltype.Ptr(S), w) - res = interpret(f, []) - assert res == lltype.nullptr(S) + res = self.interpret(f, []) + assert res == lltype.nullptr(S) -def test_weakref_simple(): - class A: - pass - class B(A): - pass - class C(A): - pass - - def f(n): - if n: - x = B() - x.hello = 42 - r = weakref.ref(x) - else: - x = C() - x.hello = 64 - r = weakref.ref(x) - return r().hello, x # returns 'x' too, to keep it alive - res = interpret(f, [1]) - assert res.item0 == 42 - res = interpret(f, [0]) - assert res.item0 == 64 - -def test_prebuilt_weakref(): - class A: - pass - a1 = A() - a1.hello = 5 - w1 = weakref.ref(a1) - a2 = A() - a2.hello = 8 - w2 = weakref.ref(a2) - - def f(n): - if n: - r = w1 - else: - r = w2 - return r().hello - res = interpret(f, [1]) - assert res == 5 - res = interpret(f, [0]) - assert res == 8 - -def test_prebuilt_dead_weakref(): - class A: - pass - a1 = A() - w1 = weakref.ref(a1) - a2 = A() - w2 = weakref.ref(a2) - - del a1 - rgc.collect() - assert w1() is None - - def f(n): - if n: - r = w1 - else: - r = w2 - return r() is not None - res = interpret(f, [1]) - assert res == False - res = interpret(f, [0]) - assert res == True +class TestOOtype(BaseTestRweakref, OORtypeMixin): + pass From arigo at codespeak.net Thu Sep 13 19:09:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Sep 2007 19:09:08 +0200 (CEST) Subject: [pypy-svn] r46548 - in pypy/dist/pypy/translator/c: . test Message-ID: <20070913170908.6EA6580B6@code0.codespeak.net> Author: arigo Date: Thu Sep 13 19:09:06 2007 New Revision: 46548 Modified: pypy/dist/pypy/translator/c/database.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/test/test_boehm.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: The (one?) framework weakref bug is that prebuilt weakrefs had their typeid not initialized - always 0. - improve the tests to expose the problem - fix The issue was caused by cfbolz being hit by a hack of mine... Replace it with a slightly more solid-looking hack. Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Thu Sep 13 19:09:06 2007 @@ -146,7 +146,7 @@ else: raise Exception("don't know about type %r" % (T,)) - def getcontainernode(self, container, **buildkwds): + def getcontainernode(self, container, _cached=True, **buildkwds): try: node = self.containernodes[container] except KeyError: @@ -156,6 +156,9 @@ self.gctransformer.consider_constant(T, container) nodefactory = ContainerNodeFactory[T.__class__] node = nodefactory(self, T, container, **buildkwds) + # _cached should only be False for a hack in weakrefnode_factory() + if not _cached: + return node self.containernodes[container] = node self.containerlist.append(node) kind = getattr(node, 'nodekind', '?') Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Thu Sep 13 19:09:06 2007 @@ -362,11 +362,7 @@ return framework.WEAKREF def convert_weakref_to(self, ptarget): - result = framework.convert_weakref_to(ptarget) - # XXX whack - gct = self.db.gctransformer - gct.gcdata.gc.gcheaderbuilder.new_header(result) - return result + return framework.convert_weakref_to(ptarget) def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Thu Sep 13 19:09:06 2007 @@ -110,6 +110,11 @@ return gc.name_to_gcpolicy[self.config.translation.gc] return self.gcpolicy + # use generate_source(defines=DEBUG_DEFINES) to force the #definition + # of the macros that enable debugging assertions + DEBUG_DEFINES = {'RPY_ASSERT': 1, + 'RPY_LL_ASSERT': 1} + def generate_source(self, db=None, defines={}): assert self.c_source_filename is None translator = self.translator Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Thu Sep 13 19:09:06 2007 @@ -865,9 +865,7 @@ ptarget = obj._dereference() wrapper = db.gcpolicy.convert_weakref_to(ptarget) container = wrapper._obj - T = typeOf(container) - nodefactory = ContainerNodeFactory[T.__class__] - return nodefactory(db, T, container) + return db.getcontainernode(container, _cached=False) ContainerNodeFactory = { Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Thu Sep 13 19:09:06 2007 @@ -34,7 +34,8 @@ t.checkgraphs() def compile(): cbuilder = CExtModuleBuilder(t, func, config=config) - c_source_filename = cbuilder.generate_source() + c_source_filename = cbuilder.generate_source( + defines = cbuilder.DEBUG_DEFINES) if conftest.option.view: t.view() cbuilder.compile() @@ -303,6 +304,7 @@ else: r = r2 a = r() + rgc.collect() if a is None: return -5 else: Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Sep 13 19:09:06 2007 @@ -614,10 +614,11 @@ result = 0 for i in range(2): a = refs[i]() + rgc.collect() if a is None: - result += i + result += (i+1) else: - result += a.hello * i + result += a.hello * (i+1) return result c_fn = self.getcompiled(fn) res = c_fn() From antocuni at codespeak.net Thu Sep 13 19:31:36 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 13 Sep 2007 19:31:36 +0200 (CEST) Subject: [pypy-svn] r46549 - in pypy/dist/pypy/translator/cli: . src Message-ID: <20070913173136.883B180C1@code0.codespeak.net> Author: antocuni Date: Thu Sep 13 19:31:34 2007 New Revision: 46549 Modified: pypy/dist/pypy/translator/cli/constant.py pypy/dist/pypy/translator/cli/cts.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: implement weakrefs for gencli Modified: pypy/dist/pypy/translator/cli/constant.py ============================================================================== --- pypy/dist/pypy/translator/cli/constant.py (original) +++ pypy/dist/pypy/translator/cli/constant.py Thu Sep 13 19:31:34 2007 @@ -396,8 +396,7 @@ class CLIWeakRefConst(CLIBaseConstMixin, WeakRefConst): def create_pointer(self, gen): - gen.ilasm.opcode('ldnull') - gen.ilasm.new('instance void %s::.ctor(object)' % self.get_type()) + gen.ilasm.new('instance void %s::.ctor()' % self.get_type()) self.db.const_count.inc('WeakRef') def get_type(self, include_class=True): @@ -406,7 +405,6 @@ def initialize_data(self, constgen, gen): if self.value is not None: push_constant(self.db, self.value._TYPE, self.value, gen) - gen.ilasm.call_method( - 'void %s::set_Target(object)' % self.get_type(), True) + gen.ilasm.call_method('void %s::ll_set(object)' % self.get_type(), True) return True Modified: pypy/dist/pypy/translator/cli/cts.py ============================================================================== --- pypy/dist/pypy/translator/cli/cts.py (original) +++ pypy/dist/pypy/translator/cli/cts.py Thu Sep 13 19:31:34 2007 @@ -104,7 +104,7 @@ char = T('char') string = T('string') - weakref = CliClassType('mscorlib', 'System.WeakReference') + weakref = CliClassType('pypylib', 'pypy.runtime.WeakReference') type = CliClassType('mscorlib', 'System.Type') object = CliClassType('mscorlib', 'System.Object') list = CliGenericType('pypylib', 'pypy.runtime.List', 1) @@ -132,7 +132,7 @@ ootype.Class: types.type, ootype.String: types.string, ootype.StringBuilder: types.string_builder, - #WeakGcAddress: types.weakref, + ootype.WeakReference: types.weakref, # maps generic types to their ordinal ootype.List.SELFTYPE_T: types.list, Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Thu Sep 13 19:31:34 2007 @@ -552,6 +552,21 @@ public TValue ll_current_value() { return it.Current.Value; } } + public class WeakReference + { + System.WeakReference wref; + + public void ll_set(object obj) + { + this.wref = new System.WeakReference(obj); + } + + public object ll_deref() + { + return this.wref.Target; + } + } + public class Record_Signed_Signed { public int item0; public int item1; From antocuni at codespeak.net Thu Sep 13 19:31:59 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 13 Sep 2007 19:31:59 +0200 (CEST) Subject: [pypy-svn] r46550 - pypy/dist/pypy/translator/oosupport Message-ID: <20070913173159.B2B2280C1@code0.codespeak.net> Author: antocuni Date: Thu Sep 13 19:31:58 2007 New Revision: 46550 Modified: pypy/dist/pypy/translator/oosupport/constant.py Log: forgot from the previous checkin Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Thu Sep 13 19:31:58 2007 @@ -25,7 +25,6 @@ by the genoo.py subclass of the backend """ -from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype import operator @@ -210,7 +209,7 @@ return genoo.CustomDictConst(self.db, value, uniq) elif isinstance(value, ootype._dict): return genoo.DictConst(self.db, value, uniq) - elif isinstance(value, llmemory.fakeweakaddress): + elif isinstance(value, ootype._weak_reference): return genoo.WeakRefConst(self.db, value, uniq) elif value is ootype.null(value._TYPE): # for NULL values, we can just use "NULL" const. This is @@ -692,8 +691,8 @@ # Weak Reference constants class WeakRefConst(AbstractConst): - def __init__(self, db, fakeaddr, count): - AbstractConst.__init__(self, db, fakeaddr.get(), count) + def __init__(self, db, wref, count): + AbstractConst.__init__(self, db, wref.ll_deref(), count) self.name = 'WEAKREF__%d' % count def OOTYPE(self): From cfbolz at codespeak.net Thu Sep 13 21:47:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Sep 2007 21:47:55 +0200 (CEST) Subject: [pypy-svn] r46556 - pypy/dist/pypy/rpython/memory Message-ID: <20070913194755.971DE80E8@code0.codespeak.net> Author: cfbolz Date: Thu Sep 13 21:47:53 2007 New Revision: 46556 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: fix generic malloc, which doesn't seem to be used often Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Thu Sep 13 21:47:53 2007 @@ -177,10 +177,10 @@ itemsize = self.varsize_item_sizes(typeid) offset_to_length = self.varsize_offset_to_length(typeid) ref = self.malloc_varsize(typeid, length, size, itemsize, - offset_to_length, True, needs_finalizer, - contains_weakptr) + offset_to_length, True, needs_finalizer) else: - ref = self.malloc_fixedsize(typeid, size, True, needs_finalizer) + ref = self.malloc_fixedsize(typeid, size, True, needs_finalizer, + contains_weakptr) # XXX lots of cast and reverse-cast around, but this malloc() # should eventually be killed return llmemory.cast_ptr_to_adr(ref) From arigo at codespeak.net Fri Sep 14 09:12:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 09:12:16 +0200 (CEST) Subject: [pypy-svn] r46562 - pypy/dist/pypy/config Message-ID: <20070914071216.0BED580E7@code0.codespeak.net> Author: arigo Date: Fri Sep 14 09:12:16 2007 New Revision: 46562 Modified: pypy/dist/pypy/config/translationoption.py Log: Sandboxing would require more work to be compatible with threads. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Fri Sep 14 09:12:16 2007 @@ -63,7 +63,8 @@ BoolOption("countmallocs", "Count mallocs and frees", default=False, cmdline=None), BoolOption("sandbox", "Produce a fully-sandboxed executable", - default=False, cmdline="--sandbox"), + default=False, cmdline="--sandbox", + requires=[("translation.thread", False)]), BoolOption("rweakref", "The backend supports RPython-level weakrefs", default=True), From arigo at codespeak.net Fri Sep 14 09:58:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 09:58:15 +0200 (CEST) Subject: [pypy-svn] r46563 - in pypy/dist/pypy: rlib rpython/lltypesystem rpython/lltypesystem/test Message-ID: <20070914075815.42C9A80E6@code0.codespeak.net> Author: arigo Date: Fri Sep 14 09:58:13 2007 New Revision: 46563 Modified: pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: Support for a custom before()/after() pair of handlers around external function calls. In this approach the pair is set up at run-time, so we get an indirect_call. The goal is both simplicity and, for the upcoming GIL-releasing code, minimal overhead when the app-level program does not use threads. Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Fri Sep 14 09:58:13 2007 @@ -148,6 +148,17 @@ def hlinvoke(repr, llcallable, *args): raise TypeError, "hlinvoke is meant to be rtyped and not called direclty" +def invoke_around_extcall(before, after): + """Call before() before any external function call, and after() after. + At the moment only one pair before()/after() can be registered at a time. + """ + if not we_are_translated(): + raise NotImplementedError("only works after rtyping") + from pypy.rpython.lltypesystem import rffi + from pypy.rpython.annlowlevel import llhelper + rffi._ll_invoke_around_extcall(llhelper(rffi.AroundFnPtr, before), + llhelper(rffi.AroundFnPtr, after)) + class UnboxedValue(object): """A mixin class to use for classes that have exactly one field which Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Fri Sep 14 09:58:13 2007 @@ -55,6 +55,7 @@ if _nowrapper: return funcptr + invoke_around_handlers = not sandboxsafe unrolling_arg_tps = unrolling_iterable(enumerate(args)) def wrapper(*args): # XXX the next line is a workaround for the annotation bug @@ -81,7 +82,13 @@ to_free = to_free + (ll_str,) else: to_free = to_free + (None,) + if invoke_around_handlers: + before = aroundstate.before + after = aroundstate.after + if before: before() result = funcptr(*real_args) + if invoke_around_handlers: + if after: after() if stringpolicy == 'fullauto': for i, tp in unrolling_arg_tps: if tp is CCHARP: @@ -92,6 +99,17 @@ wrapper._ptr = funcptr return func_with_new_name(wrapper, name) +AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) +AroundState = lltype.Struct('AroundState', + ('before', AroundFnPtr), + ('after', AroundFnPtr)) +aroundstate = lltype.malloc(AroundState, immortal=True, zero=True) +def _ll_invoke_around_extcall(before, after): + aroundstate.before = before + aroundstate.after = after + +# ____________________________________________________________ + from pypy.rpython.tool.rfficache import platform TYPES = [] Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Fri Sep 14 09:58:13 2007 @@ -359,3 +359,36 @@ assert interpret(f, [], backendopt=True) == 3 +def test_around_extcall(): + import os + from pypy.annotation import model as annmodel + from pypy.rlib.objectmodel import invoke_around_extcall + from pypy.rpython.extfuncregistry import register_external + read_fd, write_fd = os.pipe() + try: + # we need an external function that is not going to get wrapped around + # before()/after() calls, in order to call it from before()/after()... + def mywrite(s): + os.write(write_fd, s) + def llimpl(s): + s = ''.join(s.chars) + os.write(write_fd, s) + register_external(mywrite, [str], annmodel.s_None, 'll_mywrite', + llfakeimpl=llimpl, sandboxsafe=True) + + def before(): + mywrite("B") + def after(): + mywrite("A") + def f(): + os.write(write_fd, "-") + invoke_around_extcall(before, after) + os.write(write_fd, "E") + + interpret(f, []) + data = os.read(read_fd, 99) + assert data == "-BEA" + + finally: + os.close(write_fd) + os.close(read_fd) From cfbolz at codespeak.net Fri Sep 14 10:06:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Sep 2007 10:06:49 +0200 (CEST) Subject: [pypy-svn] r46564 - pypy/dist/pypy/doc Message-ID: <20070914080649.B1C6480E6@code0.codespeak.net> Author: cfbolz Date: Fri Sep 14 10:06:48 2007 New Revision: 46564 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: weakrefs are mostly properly done now! Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Fri Sep 14 10:06:48 2007 @@ -12,7 +12,6 @@ - all backends should have more consistent interfaces - geninterp is a hack - keepalives need to die, finally - - change weakrefs to work together with the GC - delegate finding type stuff like vtables etc to GC, cleaner interface for rtti, simplify translator/c/gc.py - clean up the tangle of including headers in the C backend From arigo at codespeak.net Fri Sep 14 10:33:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 10:33:08 +0200 (CEST) Subject: [pypy-svn] r46565 - in pypy/dist/pypy: module/thread module/thread/test translator/c/src translator/tool Message-ID: <20070914083308.D633980DD@code0.codespeak.net> Author: arigo Date: Fri Sep 14 10:33:07 2007 New Revision: 46565 Modified: pypy/dist/pypy/module/thread/ll_thread.py pypy/dist/pypy/module/thread/test/test_ll_thread.py pypy/dist/pypy/translator/c/src/thread.h pypy/dist/pypy/translator/c/src/thread_pthread.h pypy/dist/pypy/translator/tool/cbuild.py Log: Reintroduce fused_acquire_release_lock(), which had a good reason to exist (see comments). Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Fri Sep 14 10:33:07 2007 @@ -23,6 +23,7 @@ includes = ['unistd.h', 'src/thread.h'] def setup_thread_so(): + # XXX this is quiiiiiiiite a hack! files = [c_dir.join('src', 'thread.c')] modname = '_thread' cache_c_module(files, modname, include_dirs=[str(c_dir)]) @@ -43,6 +44,8 @@ c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT) c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void) +c_thread_fused_releaseacquirelock = llexternal( + 'RPyThreadFusedReleaseAcquireLock', [TLOCKP], lltype.Void) def allocate_lock(): ll_lock = lltype.malloc(TLOCKP.TO, flavor='raw') @@ -113,17 +116,20 @@ return bool(c_thread_acquirelock(self._lock, int(flag))) def release(self): - try: - if self.acquire(False): - # XXX the annotator trick to annotate it with non-const - # string, probably should be put into bltn-analyzers - raise error(NonConstant("bad lock")) - finally: + # Sanity check: the lock must be locked + if self.acquire(False): + c_thread_releaselock(self._lock) + raise error(NonConstant("bad lock")) + else: c_thread_releaselock(self._lock) def fused_release_acquire(self): - self.release() - self.acquire(True) + # Sanity check: the lock must be locked + if self.acquire(False): + c_thread_releaselock(self._lock) + raise error(NonConstant("bad lock")) + else: + c_thread_fused_releaseacquirelock(self._lock) def __del__(self): lltype.free(self._lock, flavor='raw') Modified: pypy/dist/pypy/module/thread/test/test_ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_ll_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_ll_thread.py Fri Sep 14 10:33:07 2007 @@ -24,6 +24,19 @@ else: py.test.fail("Did not raise") +def test_fused(): + l = allocate_lock() + try: + l.fused_release_acquire() + except error: + pass + else: + py.test.fail("Did not raise") + l.acquire(True) + l.fused_release_acquire() + could_acquire_again = l.acquire(False) + assert not could_acquire_again + def test_start_new_thread(): import time Modified: pypy/dist/pypy/translator/c/src/thread.h ============================================================================== --- pypy/dist/pypy/translator/c/src/thread.h (original) +++ pypy/dist/pypy/translator/c/src/thread.h Fri Sep 14 10:33:07 2007 @@ -39,4 +39,18 @@ #endif +/* common helper: this is a single external function so that we are + sure that nothing occurs between the release and the acquire, + e.g. no GC operation. */ + +void RPyThreadFusedReleaseAcquireLock(struct RPyOpaque_ThreadLock *lock); + +#ifndef PYPY_NOT_MAIN_FILE +void RPyThreadFusedReleaseAcquireLock(struct RPyOpaque_ThreadLock *lock) +{ + RPyThreadReleaseLock(lock); + RPyThreadAcquireLock(lock, 1); +} +#endif + #endif Modified: pypy/dist/pypy/translator/c/src/thread_pthread.h ============================================================================== --- pypy/dist/pypy/translator/c/src/thread_pthread.h (original) +++ pypy/dist/pypy/translator/c/src/thread_pthread.h Fri Sep 14 10:33:07 2007 @@ -80,10 +80,6 @@ void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag); void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); -int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock); -void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); -int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag); -void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); /* implementations */ Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Fri Sep 14 10:33:07 2007 @@ -155,8 +155,10 @@ def cache_c_module(cfiles, modname, cache_dir=None, include_dirs=None, libraries=[]): - """ Same as build c module, but instead caches results and - checks timestamps (eventually, not supported yet) + """ Same as build c module, but instead caches results. + XXX currently there is no way to force a recompile, so this is pretty + useless as soon as the sources (or headers they depend on) change :-/ + XXX for now I'm forcing a recompilation all the time. Better than not... """ from pypy.tool.autopath import pypydir if cache_dir is None: @@ -165,7 +167,7 @@ cache_dir = py.path.local(cache_dir) if cache_dir.check(dir=1): # XXX check timestamps of all cfiles - if not cache_dir.join(modname + '.so').check(): + if 1: # not cache_dir.join(modname + '.so').check(): modname = str(cache_dir.join(modname)) compile_c_module(cfiles, modname, include_dirs=include_dirs, libraries=libraries) From arigo at codespeak.net Fri Sep 14 10:37:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 10:37:10 +0200 (CEST) Subject: [pypy-svn] r46566 - in pypy/dist/pypy: module/thread translator/goal Message-ID: <20070914083710.F3E0380E7@code0.codespeak.net> Author: arigo Date: Fri Sep 14 10:37:10 2007 New Revision: 46566 Modified: pypy/dist/pypy/module/thread/gil.py pypy/dist/pypy/translator/goal/ann_override.py Log: We don't need this _annspecialcase_ any more. Modified: pypy/dist/pypy/module/thread/gil.py ============================================================================== --- pypy/dist/pypy/module/thread/gil.py (original) +++ pypy/dist/pypy/module/thread/gil.py Fri Sep 14 10:37:10 2007 @@ -40,10 +40,10 @@ """Notification that the current thread is between two bytecodes: release the GIL for a little while.""" GIL = self.GIL - GIL.release() - # Other threads can run here - GIL.acquire(True) - yield_thread._annspecialcase_ = 'specialize:yield_thread' + # Other threads can run between the release() and the acquire(). + # This is a single external function so that we are sure that nothing + # occurs between the release and the acquire, e.g. no GC operation. + GIL.fused_release_acquire() def getGIL(self): return self.GIL # XXX temporary hack! Modified: pypy/dist/pypy/translator/goal/ann_override.py ============================================================================== --- pypy/dist/pypy/translator/goal/ann_override.py (original) +++ pypy/dist/pypy/translator/goal/ann_override.py Fri Sep 14 10:37:10 2007 @@ -42,14 +42,6 @@ # clsdef = getbookkeeper().getuniqueclassdef(pycode.PyCode) # return annmodel.SomeInstance(clsdef) - def specialize__yield_thread(pol, funcdesc, args_s): - def yield_thread(self): - GIL = self.GIL - GIL.fused_release_acquire() - def builder(translator, func): - return translator.buildflowgraph(yield_thread) - return funcdesc.cachedgraph(None, builder=builder) - def specialize__wrap(pol, funcdesc, args_s): from pypy.interpreter.baseobjspace import Wrappable from pypy.annotation.classdef import ClassDef From arigo at codespeak.net Fri Sep 14 11:42:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 11:42:27 +0200 (CEST) Subject: [pypy-svn] r46568 - in pypy/dist/pypy: module/_socket module/thread module/thread/test rlib rpython/lltypesystem Message-ID: <20070914094227.4E54680EB@code0.codespeak.net> Author: arigo Date: Fri Sep 14 11:42:25 2007 New Revision: 46568 Modified: pypy/dist/pypy/module/_socket/interp_func.py pypy/dist/pypy/module/_socket/interp_socket.py pypy/dist/pypy/module/thread/gil.py pypy/dist/pypy/module/thread/ll_thread.py pypy/dist/pypy/module/thread/os_lock.py pypy/dist/pypy/module/thread/test/test_ll_thread.py pypy/dist/pypy/module/thread/test/test_lock.py pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Implement automatic GIL release around external function calls. Remove the manual GIL releases where I found them: * module/_socket * lock.acquire() Modified: pypy/dist/pypy/module/_socket/interp_func.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_func.py (original) +++ pypy/dist/pypy/module/_socket/interp_func.py Fri Sep 14 11:42:25 2007 @@ -10,12 +10,7 @@ Return the current host name. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - res = rsocket.gethostname() - finally: - if GIL is not None: GIL.acquire(True) + res = rsocket.gethostname() except SocketError, e: raise converted_error(space, e) return space.wrap(res) @@ -27,12 +22,7 @@ Return the IP address (a string of the form '255.255.255.255') for a host. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = rsocket.gethostbyname(hostname) - finally: - if GIL is not None: GIL.acquire(True) + addr = rsocket.gethostbyname(hostname) ip = addr.get_host() except SocketError, e: raise converted_error(space, e) @@ -53,12 +43,7 @@ for a host. The host argument is a string giving a host name or IP number. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - res = rsocket.gethostbyname_ex(host) - finally: - if GIL is not None: GIL.acquire(True) + res = rsocket.gethostbyname_ex(host) except SocketError, e: raise converted_error(space, e) return common_wrapgethost(space, res) @@ -71,12 +56,7 @@ for a host. The host argument is a string giving a host name or IP number. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - res = rsocket.gethostbyaddr(host) - finally: - if GIL is not None: GIL.acquire(True) + res = rsocket.gethostbyaddr(host) except SocketError, e: raise converted_error(space, e) return common_wrapgethost(space, res) @@ -94,12 +74,7 @@ else: proto = space.str_w(w_proto) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - port = rsocket.getservbyname(name, proto) - finally: - if GIL is not None: GIL.acquire(True) + port = rsocket.getservbyname(name, proto) except SocketError, e: raise converted_error(space, e) return space.wrap(port) @@ -117,12 +92,7 @@ else: proto = space.str_w(w_proto) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - service = rsocket.getservbyport(port, proto) - finally: - if GIL is not None: GIL.acquire(True) + service = rsocket.getservbyport(port, proto) except SocketError, e: raise converted_error(space, e) return space.wrap(service) @@ -134,12 +104,7 @@ Return the protocol number for the named protocol. (Rarely used.) """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - proto = rsocket.getprotobyname(name) - finally: - if GIL is not None: GIL.acquire(True) + proto = rsocket.getprotobyname(name) except SocketError, e: raise converted_error(space, e) return space.wrap(proto) @@ -150,14 +115,8 @@ Get host and port for a sockaddr.""" try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = rsocket.ipaddr_from_object(space, w_sockaddr) - host, servport = rsocket.getnameinfo(addr, flags) - finally: - if GIL is not None: GIL.acquire(True) - + addr = rsocket.ipaddr_from_object(space, w_sockaddr) + host, servport = rsocket.getnameinfo(addr, flags) except SocketError, e: raise converted_error(space, e) return space.newtuple([space.wrap(host), space.wrap(servport)]) @@ -327,14 +286,8 @@ raise OperationError(space.w_TypeError, space.wrap("Int or String expected")) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - lst = rsocket.getaddrinfo(host, port, family, socktype, - proto, flags) - finally: - if GIL is not None: GIL.acquire(True) - + lst = rsocket.getaddrinfo(host, port, family, socktype, + proto, flags) except SocketError, e: raise converted_error(space, e) lst1 = [space.newtuple([space.wrap(family), Modified: pypy/dist/pypy/module/_socket/interp_socket.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_socket.py (original) +++ pypy/dist/pypy/module/_socket/interp_socket.py Fri Sep 14 11:42:25 2007 @@ -30,12 +30,7 @@ info is a pair (hostaddr, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - sock, addr = self.accept(W_RSocket) - finally: - if GIL is not None: GIL.acquire(True) + sock, addr = self.accept(W_RSocket) return space.newtuple([space.wrap(sock), addr.as_object(space)]) except SocketError, e: @@ -50,12 +45,7 @@ sockets the address is a tuple (ifname, proto [,pkttype [,hatype]]) """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.bind(self.addr_from_object(space, w_addr)) - finally: - if GIL is not None: GIL.acquire(True) + self.bind(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) bind_w.unwrap_spec = ['self', ObjSpace, W_Root] @@ -66,12 +56,7 @@ Close the socket. It cannot be used after this call. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.close() - finally: - if GIL is not None: GIL.acquire(True) + self.close() except SocketError, e: raise converted_error(space, e) close_w.unwrap_spec = ['self', ObjSpace] @@ -83,12 +68,7 @@ is a pair (host, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.connect(self.addr_from_object(space, w_addr)) - finally: - if GIL is not None: GIL.acquire(True) + self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) except TypeError, e: @@ -102,12 +82,7 @@ This is like connect(address), but returns an error code (the errno value) instead of raising an exception when an error occurs. """ - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - error = self.connect_ex(self.addr_from_object(space, w_addr)) - finally: - if GIL is not None: GIL.acquire(True) + error = self.connect_ex(self.addr_from_object(space, w_addr)) return space.wrap(error) connect_ex_w.unwrap_spec = ['self', ObjSpace, W_Root] @@ -137,12 +112,7 @@ info is a pair (hostaddr, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = self.getpeername() - finally: - if GIL is not None: GIL.acquire(True) + addr = self.getpeername() return addr.as_object(space) except SocketError, e: raise converted_error(space, e) @@ -155,12 +125,7 @@ info is a pair (hostaddr, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = self.getsockname() - finally: - if GIL is not None: GIL.acquire(True) + addr = self.getsockname() return addr.as_object(space) except SocketError, e: raise converted_error(space, e) @@ -202,12 +167,7 @@ will allow before refusing new connections. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.listen(backlog) - finally: - if GIL is not None: GIL.acquire(True) + self.listen(backlog) except SocketError, e: raise converted_error(space, e) listen_w.unwrap_spec = ['self', ObjSpace, int] @@ -232,13 +192,7 @@ the remote end is closed and all data is read, return the empty string. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - data = self.recv(buffersize, flags) - finally: - if GIL is not None: GIL.acquire(True) - + data = self.recv(buffersize, flags) except SocketError, e: raise converted_error(space, e) return space.wrap(data) @@ -250,12 +204,7 @@ Like recv(buffersize, flags) but also return the sender's address info. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - data, addr = self.recvfrom(buffersize, flags) - finally: - if GIL is not None: GIL.acquire(True) + data, addr = self.recvfrom(buffersize, flags) if addr: w_addr = addr.as_object(space) else: @@ -274,12 +223,7 @@ """ data = coerce_to_str_w(space, w_data) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - count = self.send(data, flags) - finally: - if GIL is not None: GIL.acquire(True) + count = self.send(data, flags) except SocketError, e: raise converted_error(space, e) return space.wrap(count) @@ -295,12 +239,7 @@ """ data = coerce_to_str_w(space, w_data) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - count = self.sendall(data, flags) - finally: - if GIL is not None: GIL.acquire(True) + count = self.sendall(data, flags) except SocketError, e: raise converted_error(space, e) sendall_w.unwrap_spec = ['self', ObjSpace, W_Root, int] @@ -321,13 +260,8 @@ flags = space.int_w(w_param2) w_addr = w_param3 try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = self.addr_from_object(space, w_addr) - count = self.sendto(data, flags, addr) - finally: - if GIL is not None: GIL.acquire(True) + addr = self.addr_from_object(space, w_addr) + count = self.sendto(data, flags, addr) except SocketError, e: raise converted_error(space, e) return space.wrap(count) @@ -391,12 +325,7 @@ (flag == SHUT_RDWR). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.shutdown(how) - finally: - if GIL is not None: GIL.acquire(True) + self.shutdown(how) except SocketError, e: raise converted_error(space, e) shutdown_w.unwrap_spec = ['self', ObjSpace, int] Modified: pypy/dist/pypy/module/thread/gil.py ============================================================================== --- pypy/dist/pypy/module/thread/gil.py (original) +++ pypy/dist/pypy/module/thread/gil.py Fri Sep 14 11:42:25 2007 @@ -10,6 +10,7 @@ from pypy.module.thread import ll_thread as thread from pypy.interpreter.miscutils import Action from pypy.module.thread.threadlocals import OSThreadLocals +from pypy.rlib.objectmodel import invoke_around_extcall class GILThreadLocals(OSThreadLocals): """A version of OSThreadLocals that enforces a GIL.""" @@ -18,13 +19,25 @@ def setup_threads(self, space): """Enable threads in the object space, if they haven't already been.""" if self.GIL is None: - self.GIL = thread.allocate_lock() + self.GIL = thread.allocate_lock_NOAUTO() self.enter_thread(space) # setup the main thread # add the GIL-releasing callback as an action on the space space.pending_actions.append(GILReleaseAction(self)) - return True + result = True else: - return False # already set up + result = False # already set up + + # add the GIL-releasing callback around external function calls. + # + # XXX we assume a single space, but this is not quite true during + # testing; for example, if you run the whole of test_lock you get + # a deadlock caused by the first test's space being reused by + # test_lock_again after the global state was cleared by + # test_compile_lock. As a workaround, we repatch these global + # fields systematically. + spacestate.GIL = self.GIL + invoke_around_extcall(before_external_call, after_external_call) + return result def enter_thread(self, space): "Notification that the current thread is just starting: grab the GIL." @@ -60,3 +73,14 @@ def perform(self): self.threadlocals.yield_thread() + + +class SpaceState: + pass +spacestate = SpaceState() + +def before_external_call(): + spacestate.GIL.release() + +def after_external_call(): + spacestate.GIL.acquire(True) Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Fri Sep 14 11:42:25 2007 @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.lltype import typeOf -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import debug_assert from pypy.rlib.nonconst import NonConstant class error(Exception): @@ -30,9 +30,10 @@ return str(pypydir.join('_cache', modname)) + '.so' libraries = [setup_thread_so()] -def llexternal(name, args, result): +def llexternal(name, args, result, **kwds): return rffi.llexternal(name, args, result, includes=includes, - libraries=libraries, include_dirs=[str(c_dir)]) + libraries=libraries, include_dirs=[str(c_dir)], + **kwds) CALLBACK = lltype.Ptr(lltype.FuncType([rffi.VOIDP], rffi.VOIDP)) c_thread_start = llexternal('RPyThreadStart', [CALLBACK, rffi.VOIDP], rffi.INT) @@ -44,8 +45,18 @@ c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT) c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void) -c_thread_fused_releaseacquirelock = llexternal( - 'RPyThreadFusedReleaseAcquireLock', [TLOCKP], lltype.Void) + +# another set of functions, this time in versions that don't cause the +# GIL to be released. To use to handle the GIL lock itself. +c_thread_acquirelock_NOAUTO = llexternal('RPyThreadAcquireLock', + [TLOCKP, rffi.INT], rffi.INT, + sandboxsafe=True) +c_thread_releaselock_NOAUTO = llexternal('RPyThreadReleaseLock', + [TLOCKP], lltype.Void, + sandboxsafe=True) +c_thread_fused_releaseacquirelock_NOAUTO = llexternal( + 'RPyThreadFusedReleaseAcquireLock', [TLOCKP], lltype.Void, + sandboxsafe=True) def allocate_lock(): ll_lock = lltype.malloc(TLOCKP.TO, flavor='raw') @@ -55,6 +66,14 @@ raise error("out of resources") return Lock(ll_lock) +def allocate_lock_NOAUTO(): + ll_lock = lltype.malloc(TLOCKP.TO, flavor='raw') + res = c_thread_lock_init(ll_lock) + if res == -1: + lltype.free(ll_lock, flavor='raw') + raise error("out of resources") + return Lock_NOAUTO(ll_lock) + def _start_new_thread(x, y): return thread.start_new_thread(x, (y,)) @@ -123,14 +142,26 @@ else: c_thread_releaselock(self._lock) + def __del__(self): + lltype.free(self._lock, flavor='raw') + +class Lock_NOAUTO(object): + """A special lock that doesn't cause the GIL to be released when + we try to acquire it. Used for the GIL itself.""" + + def __init__(self, ll_lock): + self._lock = ll_lock + + def acquire(self, flag): + return bool(c_thread_acquirelock_NOAUTO(self._lock, int(flag))) + + def release(self): + debug_assert(not self.acquire(False), "Lock_NOAUTO was not held!") + c_thread_releaselock_NOAUTO(self._lock) + def fused_release_acquire(self): - # Sanity check: the lock must be locked - if self.acquire(False): - c_thread_releaselock(self._lock) - raise error(NonConstant("bad lock")) - else: - c_thread_fused_releaseacquirelock(self._lock) + debug_assert(not self.acquire(False), "Lock_NOAUTO was not held!") + c_thread_fused_releaseacquirelock_NOAUTO(self._lock) def __del__(self): lltype.free(self._lock, flavor='raw') - Modified: pypy/dist/pypy/module/thread/os_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/os_lock.py (original) +++ pypy/dist/pypy/module/thread/os_lock.py Fri Sep 14 11:42:25 2007 @@ -39,15 +39,8 @@ With an argument, this will only block if the argument is true, and the return value reflects whether the lock is acquired. The blocking operation is not interruptible.""" - # XXX Usage of threadlocals.GIL in this function is considered hackish. - # Ideally, all GIL knowledge should be in gil.py. mylock = self.lock - GIL = space.threadlocals.getGIL() - if GIL is not None: - GIL.release() result = mylock.acquire(bool(waitflag)) - if GIL is not None: - GIL.acquire(True) return space.newbool(result) def descr_lock_release(self, space): Modified: pypy/dist/pypy/module/thread/test/test_ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_ll_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_ll_thread.py Fri Sep 14 11:42:25 2007 @@ -25,17 +25,14 @@ py.test.fail("Did not raise") def test_fused(): - l = allocate_lock() - try: - l.fused_release_acquire() - except error: - pass - else: - py.test.fail("Did not raise") + l = allocate_lock_NOAUTO() l.acquire(True) l.fused_release_acquire() could_acquire_again = l.acquire(False) assert not could_acquire_again + l.release() + could_acquire_again = l.acquire(False) + assert could_acquire_again def test_start_new_thread(): import time Modified: pypy/dist/pypy/module/thread/test/test_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_lock.py (original) +++ pypy/dist/pypy/module/thread/test/test_lock.py Fri Sep 14 11:42:25 2007 @@ -1,4 +1,5 @@ from pypy.module.thread.test.support import GenericTestThread +from pypy.translator.c.test.test_genc import compile class AppTestLock(GenericTestThread): @@ -27,3 +28,29 @@ lock.acquire() assert lock.locked() is True assert feedback == [42] + + +def test_compile_lock(): + from pypy.rlib import rgc + from pypy.module.thread.ll_thread import allocate_lock + def g(): + l = allocate_lock() + ok1 = l.acquire(True) + ok2 = l.acquire(False) + l.release() + ok3 = l.acquire(False) + res = ok1 and not ok2 and ok3 + return res + g.dont_inline = True + def f(): + res = g() + # the lock must have been freed by now - we use refcounting + return res + fn = compile(f, [], gcpolicy='ref') + res = fn() + assert res + + +class AppTestLockAgain(GenericTestThread): + # test it at app-level again to detect strange interactions + test_lock_again = AppTestLock.test_lock.im_func Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Fri Sep 14 11:42:25 2007 @@ -152,12 +152,12 @@ """Call before() before any external function call, and after() after. At the moment only one pair before()/after() can be registered at a time. """ - if not we_are_translated(): - raise NotImplementedError("only works after rtyping") + # NOTE: the hooks are cleared during translation! To be effective + # in a compiled program they must be set at run-time. from pypy.rpython.lltypesystem import rffi from pypy.rpython.annlowlevel import llhelper - rffi._ll_invoke_around_extcall(llhelper(rffi.AroundFnPtr, before), - llhelper(rffi.AroundFnPtr, after)) + rffi.aroundstate.before = llhelper(rffi.AroundFnPtr, before) + rffi.aroundstate.after = llhelper(rffi.AroundFnPtr, after) class UnboxedValue(object): Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Fri Sep 14 11:42:25 2007 @@ -100,13 +100,13 @@ return func_with_new_name(wrapper, name) AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -AroundState = lltype.Struct('AroundState', - ('before', AroundFnPtr), - ('after', AroundFnPtr)) -aroundstate = lltype.malloc(AroundState, immortal=True, zero=True) -def _ll_invoke_around_extcall(before, after): - aroundstate.before = before - aroundstate.after = after +class AroundState: + def _freeze_(self): + self.before = lltype.nullptr(AroundFnPtr.TO) + self.after = lltype.nullptr(AroundFnPtr.TO) + return False +aroundstate = AroundState() +aroundstate._freeze_() # ____________________________________________________________ From antocuni at codespeak.net Fri Sep 14 11:53:28 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 14 Sep 2007 11:53:28 +0200 (CEST) Subject: [pypy-svn] r46569 - in pypy/dist/pypy: annotation rpython rpython/test Message-ID: <20070914095328.F0B2880A0@code0.codespeak.net> Author: antocuni Date: Fri Sep 14 11:53:28 2007 New Revision: 46569 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/rpython/rweakref.py pypy/dist/pypy/rpython/test/test_rweakref.py Log: allow null weakref in both lltype and ootype Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Fri Sep 14 11:53:28 2007 @@ -765,6 +765,8 @@ _make_none_union('SomeList', 'obj.listdef') _make_none_union('SomeDict', 'obj.dictdef') _make_none_union('SomeExternalObject', 'obj.knowntype') +_make_none_union('SomeWeakRef', 'obj.classdef') +_make_none_union('SomeDeadWeakRef', '') # getitem on SomePBCs, in particular None fails Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Fri Sep 14 11:53:28 2007 @@ -500,16 +500,10 @@ def __init__(self, classdef): self.classdef = classdef - def can_be_none(self): - return False - class SomeDeadWeakRef(SomeObject): knowntype = weakref.ReferenceType immutable = True - def can_be_none(self): - return False - # ____________________________________________________________ # memory addresses Modified: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/rweakref.py (original) +++ pypy/dist/pypy/rpython/rweakref.py Fri Sep 14 11:53:28 2007 @@ -29,6 +29,9 @@ "this backend or GC policy") def convert_const(self, value): + if value is None: + return self.null_wref + assert isinstance(value, weakref.ReferenceType) instance = value() bk = self.rtyper.annotator.bookkeeper @@ -47,6 +50,7 @@ class LLWeakRefRepr(BaseWeakRefRepr): lowleveltype = llmemory.WeakRefPtr dead_wref = llmemory.dead_wref + null_wref = lltype.nullptr(llmemory.WeakRef) def rtype_simple_call(self, hop): v_wref, = hop.inputargs(self) @@ -59,6 +63,7 @@ class OOWeakRefRepr(BaseWeakRefRepr): lowleveltype = ootype.WeakReference dead_wref = ootype.dead_wref + null_wref = ootype.null(ootype.WeakReference) def rtype_simple_call(self, hop): v_wref, = hop.inputargs(self) Modified: pypy/dist/pypy/rpython/test/test_rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rweakref.py (original) +++ pypy/dist/pypy/rpython/test/test_rweakref.py Fri Sep 14 11:53:28 2007 @@ -72,6 +72,16 @@ res = self.interpret(f, [0]) assert res == True + def test_pbc_null_weakref(self): + class A: + pass + a1 = A() + mylist = [weakref.ref(a1), None] + def fn(i): + item = mylist[i] + return item is None + assert self.interpret(fn, [0]) is False + assert self.interpret(fn, [1]) is True class TestLLtype(BaseTestRweakref, LLRtypeMixin): def test_ll_weakref(self): From antocuni at codespeak.net Fri Sep 14 12:01:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 14 Sep 2007 12:01:00 +0200 (CEST) Subject: [pypy-svn] r46571 - pypy/dist/pypy/translator/oosupport Message-ID: <20070914100100.916CC80EB@code0.codespeak.net> Author: antocuni Date: Fri Sep 14 12:01:00 2007 New Revision: 46571 Modified: pypy/dist/pypy/translator/oosupport/constant.py Log: don't crash when rendering prebuilt null weakrefs Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Fri Sep 14 12:01:00 2007 @@ -692,7 +692,11 @@ class WeakRefConst(AbstractConst): def __init__(self, db, wref, count): - AbstractConst.__init__(self, db, wref.ll_deref(), count) + if wref: + value = wref.ll_deref() + else: + value = None + AbstractConst.__init__(self, db, value, count) self.name = 'WEAKREF__%d' % count def OOTYPE(self): @@ -700,7 +704,7 @@ return None def is_null(self): - return False + return self.value is None def record_dependencies(self): if self.value is not None: From antocuni at codespeak.net Fri Sep 14 12:03:07 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 14 Sep 2007 12:03:07 +0200 (CEST) Subject: [pypy-svn] r46572 - pypy/dist/pypy/translator/jvm Message-ID: <20070914100307.700E080F6@code0.codespeak.net> Author: antocuni Date: Fri Sep 14 12:03:06 2007 New Revision: 46572 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/node.py Log: implement branch_if_not_equal for genjvm; this is needed by backendopts. Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Fri Sep 14 12:03:06 2007 @@ -998,6 +998,9 @@ def branch_if_equal(self, target_label): self._instr(IF_ICMPEQ, target_label) + def branch_if_not_equal(self, target_label): + self._instr(IF_ICMPNE, target_label) + def call_graph(self, graph): mthd = self.db.pending_function(graph) mthd.invoke(self) Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Fri Sep 14 12:03:06 2007 @@ -242,6 +242,9 @@ self.jrettype = jrettype self._block_labels = {} + def next_label(self, prefix='label'): + return self.generator.unique_label(prefix) + def method(self): """ Returns a jvmgen.Method that can invoke this function """ if not self.is_method: From arigo at codespeak.net Fri Sep 14 12:15:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 12:15:27 +0200 (CEST) Subject: [pypy-svn] r46574 - in pypy/dist/pypy: annotation rpython rpython/test Message-ID: <20070914101527.4794980FB@code0.codespeak.net> Author: arigo Date: Fri Sep 14 12:15:26 2007 New Revision: 46574 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/rpython/rweakref.py pypy/dist/pypy/rpython/test/test_rweakref.py Log: Replace SomeDeadWeakRef() with SomeWeakRef(classdef=None). Proper support for known-to-be-dead weakrefs in rweakref. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Fri Sep 14 12:15:26 2007 @@ -11,8 +11,7 @@ from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator from pypy.annotation.model import SomePBC, SomeSlice, SomeFloat, s_None -from pypy.annotation.model import SomeExternalObject -from pypy.annotation.model import SomeWeakRef, SomeDeadWeakRef +from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeCTypesObject from pypy.annotation.model import unionof, UnionError, set, missing_operation, TLS @@ -766,7 +765,6 @@ _make_none_union('SomeDict', 'obj.dictdef') _make_none_union('SomeExternalObject', 'obj.knowntype') _make_none_union('SomeWeakRef', 'obj.classdef') -_make_none_union('SomeDeadWeakRef', '') # getitem on SomePBCs, in particular None fails @@ -873,23 +871,16 @@ class __extend__(pairtype(SomeWeakRef, SomeWeakRef)): def union((s_wrf1, s_wrf2)): - basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) - if basedef is None: - return SomeObject() + if s_wrf1.classdef is None: + basedef = s_wrf2.classdef # s_wrf1 is known to be dead + elif s_wrf2.classdef is None: + basedef = s_wrf1.classdef # s_wrf2 is known to be dead + else: + basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) + if basedef is None: # no common base class! complain... + return SomeObject() return SomeWeakRef(basedef) -class __extend__(pairtype(SomeWeakRef, SomeDeadWeakRef)): - def union((s_wrf1, s_wrf2)): - return SomeWeakRef(s_wrf1.classdef) - -class __extend__(pairtype(SomeDeadWeakRef, SomeWeakRef)): - def union((s_wrf1, s_wrf2)): - return SomeWeakRef(s_wrf2.classdef) - -class __extend__(pairtype(SomeDeadWeakRef, SomeDeadWeakRef)): - def union((s_wrf1, s_wrf2)): - return SomeDeadWeakRef() - #_________________________________________ # memory addresses Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Fri Sep 14 12:15:26 2007 @@ -11,7 +11,7 @@ SomeInteger, SomeExternalObject, SomeOOInstance, TLS, SomeAddress, \ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ - SomeList, SomeObject, HarmlesslyBlocked, SomeWeakRef, SomeDeadWeakRef + SomeList, SomeObject, HarmlesslyBlocked, SomeWeakRef from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -391,7 +391,7 @@ elif tp is weakref.ReferenceType: x1 = x() if x1 is None: - result = SomeDeadWeakRef() + result = SomeWeakRef(None) # dead weakref else: s1 = self.immutablevalue(x1) assert isinstance(s1, SomeInstance) Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Fri Sep 14 12:15:26 2007 @@ -498,12 +498,9 @@ knowntype = weakref.ReferenceType immutable = True def __init__(self, classdef): + # 'classdef' is None for known-to-be-dead weakrefs. self.classdef = classdef -class SomeDeadWeakRef(SomeObject): - knowntype = weakref.ReferenceType - immutable = True - # ____________________________________________________________ # memory addresses Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Fri Sep 14 12:15:26 2007 @@ -9,7 +9,7 @@ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ SomeCTypesObject, s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ - SomeGenericCallable, SomeWeakRef, SomeDeadWeakRef + SomeGenericCallable, SomeWeakRef from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? @@ -755,11 +755,10 @@ class __extend__(SomeWeakRef): def simple_call(s_wrf): - return SomeInstance(s_wrf.classdef, can_be_None=True) - -class __extend__(SomeDeadWeakRef): - def simple_call(s_wrf): - return s_None + if s_wrf.classdef is None: + return s_None # known to be a dead weakref + else: + return SomeInstance(s_wrf.classdef, can_be_None=True) #_________________________________________ # memory addresses Modified: pypy/dist/pypy/rpython/rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/rweakref.py (original) +++ pypy/dist/pypy/rpython/rweakref.py Fri Sep 14 12:15:26 2007 @@ -55,7 +55,11 @@ def rtype_simple_call(self, hop): v_wref, = hop.inputargs(self) hop.exception_cannot_occur() - return hop.genop('weakref_deref', [v_wref], resulttype=hop.r_result) + if hop.r_result.lowleveltype is lltype.Void: # known-to-be-dead weakref + return hop.inputconst(lltype.Void, None) + else: + return hop.genop('weakref_deref', [v_wref], + resulttype=hop.r_result) def _weakref_create(self, llinstance): return llmemory.weakref_create(llinstance) @@ -69,8 +73,12 @@ v_wref, = hop.inputargs(self) cname = hop.inputconst(ootype.Void, 'll_deref') hop.exception_cannot_occur() - v_deref = hop.genop('oosend', [cname, v_wref], resulttype=ootype.ROOT) - return hop.genop('oodowncast', [v_deref], resulttype=hop.r_result) + if hop.r_result.lowleveltype is lltype.Void: # known-to-be-dead weakref + return hop.inputconst(lltype.Void, None) + else: + v_deref = hop.genop('oosend', [cname, v_wref], + resulttype=ootype.ROOT) + return hop.genop('oodowncast', [v_deref], resulttype=hop.r_result) def _weakref_create(self, llinstance): return ootype.ooweakref_create(llinstance) Modified: pypy/dist/pypy/rpython/test/test_rweakref.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rweakref.py (original) +++ pypy/dist/pypy/rpython/test/test_rweakref.py Fri Sep 14 12:15:26 2007 @@ -72,6 +72,46 @@ res = self.interpret(f, [0]) assert res == True + def test_multiple_prebuilt_dead_weakrefs(self): + class A: + pass + a1 = A() + w1 = weakref.ref(a1) + a2 = A() + w2 = weakref.ref(a2) + a3 = A() + w3 = weakref.ref(a3) + a4 = A() + w4 = weakref.ref(a4) + + del a1, a3 + rgc.collect() + assert w1() is None + assert w3() is None + + def f(n): + if n > 0: + if n > 5: + r = w1 + else: + r = w3 + assert r() is None + else: + if n < -5: + r = w2 + else: + r = w4 + assert r() is not None + return r() is not None + res = self.interpret(f, [1]) + assert res == False + res = self.interpret(f, [0]) + assert res == True + res = self.interpret(f, [100]) + assert res == False + res = self.interpret(f, [-100]) + assert res == True + def test_pbc_null_weakref(self): class A: pass From cfbolz at codespeak.net Fri Sep 14 13:32:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Sep 2007 13:32:23 +0200 (CEST) Subject: [pypy-svn] r46575 - in pypy/dist/pypy: rpython/ootypesystem translator/backendopt translator/backendopt/test Message-ID: <20070914113223.874F680EE@code0.codespeak.net> Author: cfbolz Date: Fri Sep 14 13:32:22 2007 New Revision: 46575 Modified: pypy/dist/pypy/rpython/ootypesystem/rstr.py pypy/dist/pypy/translator/backendopt/inline.py pypy/dist/pypy/translator/backendopt/test/test_inline.py Log: issue313 resolved Fix the inlining bug: the inliner only considered calls to be raising in one place. Modified: pypy/dist/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rstr.py Fri Sep 14 13:32:22 2007 @@ -202,11 +202,7 @@ return sign * val def ll_float(ll_str): - # XXX workaround for an inlining bug - try: - return ootype.ooparse_float(ll_str) - except ValueError: - raise + return ootype.ooparse_float(ll_str) # interface to build strings: # x = ll_build_start(n) Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Fri Sep 14 13:32:22 2007 @@ -162,6 +162,14 @@ return True else: return True # conservatively + for block in from_graph.iterblocks(): + if block.exitswitch == c_last_exception: + consider_ops_to = -1 + else: + consider_ops_to = len(block.operations) + for op in block.operations[:consider_ops_to]: + if raise_analyzer.can_raise(op): + return True return False class BaseInliner(object): Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_inline.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_inline.py Fri Sep 14 13:32:22 2007 @@ -65,7 +65,9 @@ t.buildrtyper(type_system=self.type_system).specialize() return t - def check_inline(self, func, in_func, sig, entry=None, inline_guarded_calls=False): + def check_inline(self, func, in_func, sig, entry=None, + inline_guarded_calls=False, + graph=False): if entry is None: entry = in_func t = self.translate(entry, sig) @@ -85,6 +87,8 @@ interp = LLInterpreter(t.rtyper) def eval_func(args): return interp.eval_graph(graphof(t, entry), args) + if graph: + return eval_func, graphof(t, func) return eval_func def check_auto_inlining(self, func, sig, multiplier=None, call_count_check=False, @@ -280,6 +284,19 @@ result = eval_func([2]) assert result == 87 + def test_inline_with_raising_non_call_op(self): + class A: + pass + def f(): + return A() + def g(): + try: + a = f() + except MemoryError: + return 1 + return 2 + py.test.raises(CannotInline, self.check_inline, f, g, []) + def test_inline_var_exception(self): def f(x): e = None From antocuni at codespeak.net Fri Sep 14 14:33:39 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 14 Sep 2007 14:33:39 +0200 (CEST) Subject: [pypy-svn] r46579 - in pypy/dist/pypy/config: . test Message-ID: <20070914123339.D576080F1@code0.codespeak.net> Author: antocuni Date: Fri Sep 14 14:33:39 2007 New Revision: 46579 Modified: pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/config/test/test_pypyoption.py pypy/dist/pypy/config/translationoption.py Log: - enable rweakref with ootype - disable objspace options that requires rweakref when it's disabled; doesn't work, though :-( Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Fri Sep 14 14:33:39 2007 @@ -187,18 +187,21 @@ BoolOption("withtypeversion", "version type objects when changing them", cmdline=None, - default=False), + default=False, + requires=[("translation.rweakref", True)]), BoolOption("withshadowtracking", "track whether an instance attribute shadows a type" " attribute", default=False, requires=[("objspace.std.withmultidict", True), - ("objspace.std.withtypeversion", True)]), + ("objspace.std.withtypeversion", True), + ("translation.rweakref", False)]), BoolOption("withmethodcache", "try to cache method lookups", default=False, - requires=[("objspace.std.withtypeversion", True)]), + requires=[("objspace.std.withtypeversion", True), + ("translation.rweakref", False)]), BoolOption("withmethodcachecounter", "try to cache methods and provide a counter in __pypy__. " "for testing purposes only.", Modified: pypy/dist/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/dist/pypy/config/test/test_pypyoption.py (original) +++ pypy/dist/pypy/config/test/test_pypyoption.py Fri Sep 14 14:33:39 2007 @@ -31,3 +31,13 @@ for path in c.getpaths(include_groups=True): fn = prefix + "." + path + ".txt" assert configdocdir.join(fn).check() + +def test_rweakref_required(): + py.test.skip('fixme!') + conf = get_pypy_config() + conf.translation.rweakref = False + conf.objspace.std.allopts = True + + assert not conf.objspace.std.withtypeversion + assert not conf.objspace.std.withmethodcache + assert not conf.objspace.std.withshadowtracking Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Fri Sep 14 14:33:39 2007 @@ -21,7 +21,7 @@ ("translation.backendopt.constfold", False), ("translation.backendopt.heap2stack", False), ("translation.backendopt.clever_malloc_removal", False), - ("translation.rweakref", False)], # XXX + ] }), ChoiceOption("backend", "Backend to use for code generation", ["c", "llvm", "cli", "jvm", "js", "squeak", "cl"], From antocuni at codespeak.net Fri Sep 14 14:45:13 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 14 Sep 2007 14:45:13 +0200 (CEST) Subject: [pypy-svn] r46580 - in pypy/dist/pypy/translator: cli/test jvm jvm/test oosupport/test_template Message-ID: <20070914124513.B606680F7@code0.codespeak.net> Author: antocuni Date: Fri Sep 14 14:45:10 2007 New Revision: 46580 Added: pypy/dist/pypy/translator/jvm/test/test_backendopt.py (contents, props changed) pypy/dist/pypy/translator/oosupport/test_template/backendopt.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/test/test_backendopt.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/jvm/test/runtest.py Log: move test_backendopt to oosupport, and port it to genvjm Modified: pypy/dist/pypy/translator/cli/test/test_backendopt.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_backendopt.py (original) +++ pypy/dist/pypy/translator/cli/test/test_backendopt.py Fri Sep 14 14:45:10 2007 @@ -1,27 +1,7 @@ import py from pypy.translator.cli.test.runtest import compile_function -from pypy.translator.c.test.test_backendoptimized import \ - TestTypedOptimizedSwitchTestCase as c_TestTypedOptimizedSwitchTestCase - -class CTestCompat: - def CodeGenerator(self): - return self +from pypy.translator.oosupport.test_template.backendopt import BaseTestOptimizedSwitch +class TestOptimizedSwitch(BaseTestOptimizedSwitch): def getcompiled(self, fn, annotation): return compile_function(fn, annotation, backendopt=True) - -class TestOptimizedSwitchTestCase(CTestCompat, c_TestTypedOptimizedSwitchTestCase): - - def test_switch_naive(self): - def fn(x): - if x == -1: - return 3 - elif x == 3: - return 9 - elif x == 9: - return -1 - return 42 - codegenerator = self.CodeGenerator() - fn = codegenerator.getcompiled(fn, [int]) - for x in (-5,-1,0,3,9,27,48): - assert fn(x) == fn(x) Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Fri Sep 14 14:45:10 2007 @@ -1378,3 +1378,8 @@ def get_instruction_count(self): return self.curfunc.instr_counter + + def emit_tableswitch(self, low, lbls, default): + pass + + Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Fri Sep 14 14:45:10 2007 @@ -9,6 +9,8 @@ from pypy.tool.udir import udir from pypy.translator.translator import TranslationContext from pypy.translator.oosupport.genoo import GenOO +from pypy.translator.backendopt.all import backend_optimizations +from pypy.translator.backendopt.checkvirtual import check_virtual_methods from pypy.translator.jvm.generator import JasminGenerator from pypy.translator.jvm.option import getoption @@ -202,7 +204,7 @@ sys.stderr.write(stderr) return stdout -def generate_source_for_function(func, annotation): +def generate_source_for_function(func, annotation, backendopt=False): """ Given a Python function and some hints about its argument types, @@ -216,6 +218,9 @@ ann = t.buildannotator() ann.build_types(func, annotation) t.buildrtyper(type_system="ootype").specialize() + if backendopt: + check_virtual_methods(ootype.ROOT) + backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() if getoption('wd'): tmpdir = py.path.local('.') Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Fri Sep 14 14:45:10 2007 @@ -84,7 +84,7 @@ self._ann = None self._jvm_src = None - def _compile(self, fn, args, ann=None): + def _compile(self, fn, args, ann=None, backendopt=False): if ann is None: ann = [lltype_to_annotation(typeOf(x)) for x in args] if self._func is fn and self._ann == ann: @@ -93,7 +93,7 @@ self._func = fn self._ann = ann olddefs = patch_os() - self._jvm_src = generate_source_for_function(fn, ann) + self._jvm_src = generate_source_for_function(fn, ann, backendopt) unpatch_os(olddefs) if not getoption('noasm'): self._jvm_src.compile() Added: pypy/dist/pypy/translator/jvm/test/test_backendopt.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_backendopt.py Fri Sep 14 14:45:10 2007 @@ -0,0 +1,8 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.translator.oosupport.test_template.backendopt import BaseTestOptimizedSwitch + +class TestOptimizedSwitch(BaseTestOptimizedSwitch): + def getcompiled(self, fn, annotation): + t = JvmTest() + return t._compile(fn, None, annotation, backendopt=True) Added: pypy/dist/pypy/translator/oosupport/test_template/backendopt.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/test_template/backendopt.py Fri Sep 14 14:45:10 2007 @@ -0,0 +1,25 @@ +import py +from pypy.translator.c.test.test_backendoptimized import \ + TestTypedOptimizedSwitchTestCase as c_TestTypedOptimizedSwitchTestCase + +class BaseTestOptimizedSwitch(c_TestTypedOptimizedSwitchTestCase): + + def CodeGenerator(self): + return self + + def getcompiled(self, fn, annotation): + return compile_function(fn, annotation, backendopt=True) + + def test_switch_naive(self): + def fn(x): + if x == -1: + return 3 + elif x == 3: + return 9 + elif x == 9: + return -1 + return 42 + codegenerator = self.CodeGenerator() + fn = codegenerator.getcompiled(fn, [int]) + for x in (-5,-1,0,3,9,27,48): + assert fn(x) == fn(x) From cfbolz at codespeak.net Fri Sep 14 14:54:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Sep 2007 14:54:58 +0200 (CEST) Subject: [pypy-svn] r46581 - in pypy/dist/pypy/config: . test Message-ID: <20070914125458.755B880F7@code0.codespeak.net> Author: cfbolz Date: Fri Sep 14 14:54:56 2007 New Revision: 46581 Modified: pypy/dist/pypy/config/config.py pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/config/test/test_config.py pypy/dist/pypy/config/test/test_pypyoption.py Log: better suggestion handling Modified: pypy/dist/pypy/config/config.py ============================================================================== --- pypy/dist/pypy/config/config.py (original) +++ pypy/dist/pypy/config/config.py Fri Sep 14 14:54:56 2007 @@ -267,12 +267,18 @@ for path, reqvalue in self._requires: toplevel = config._cfgimpl_get_toplevel() homeconfig, name = toplevel._cfgimpl_get_home_by_path(path) - homeconfig.setoption(name, reqvalue, who) + homeconfig.setoption(name, reqvalue, "required") if value and self._suggests is not None: for path, reqvalue in self._suggests: toplevel = config._cfgimpl_get_toplevel() homeconfig, name = toplevel._cfgimpl_get_home_by_path(path) - homeconfig.setoption(name, reqvalue, "suggested") + try: + homeconfig.setoption(name, reqvalue, "suggested") + except ValueError: + # setting didn't work, but that is fine, since it is + # suggested + pass + super(BoolOption, self).setoption(config, value, who) def add_optparse_option(self, argnames, parser, config): Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Fri Sep 14 14:54:56 2007 @@ -196,12 +196,12 @@ default=False, requires=[("objspace.std.withmultidict", True), ("objspace.std.withtypeversion", True), - ("translation.rweakref", False)]), + ("translation.rweakref", True)]), BoolOption("withmethodcache", "try to cache method lookups", default=False, requires=[("objspace.std.withtypeversion", True), - ("translation.rweakref", False)]), + ("translation.rweakref", True)]), BoolOption("withmethodcachecounter", "try to cache methods and provide a counter in __pypy__. " "for testing purposes only.", Modified: pypy/dist/pypy/config/test/test_config.py ============================================================================== --- pypy/dist/pypy/config/test/test_config.py (original) +++ pypy/dist/pypy/config/test/test_config.py Fri Sep 14 14:54:56 2007 @@ -471,6 +471,41 @@ assert c.opt assert not c.toplevel +def test_suggests_can_fail(): + descr = OptionDescription("test", '', [ + BoolOption("t1", "", default=False), + BoolOption("t2", "", default=False, + requires=[("t3", True)]), + BoolOption("t3", "", default=False), + BoolOption("opt", "", default=False, + suggests=[("t1", True), ("t2", True)]) + ]) + c = Config(descr) + assert not c.t1 + assert not c.t2 + assert not c.t3 + assert not c.opt + c.opt = True + assert c.opt + assert c.t1 + assert c.t2 + assert c.t3 + # does not crash + c.t2 = False + assert not c.t2 + + c = Config(descr) + c.t3 = False + assert not c.t3 + # does not crash + c.opt = True + assert c.opt + assert not c.t3 + assert not c.t2 + + + + def test_delattr(): descr = OptionDescription("opt", "", [ OptionDescription("s1", "", [ Modified: pypy/dist/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/dist/pypy/config/test/test_pypyoption.py (original) +++ pypy/dist/pypy/config/test/test_pypyoption.py Fri Sep 14 14:54:56 2007 @@ -33,7 +33,6 @@ assert configdocdir.join(fn).check() def test_rweakref_required(): - py.test.skip('fixme!') conf = get_pypy_config() conf.translation.rweakref = False conf.objspace.std.allopts = True From arigo at codespeak.net Fri Sep 14 15:27:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 15:27:43 +0200 (CEST) Subject: [pypy-svn] r46582 - pypy/dist/pypy/translator/c/src Message-ID: <20070914132743.751F080DD@code0.codespeak.net> Author: arigo Date: Fri Sep 14 15:27:43 2007 New Revision: 46582 Modified: pypy/dist/pypy/translator/c/src/thread.c Log: Style. Modified: pypy/dist/pypy/translator/c/src/thread.c ============================================================================== --- pypy/dist/pypy/translator/c/src/thread.c (original) +++ pypy/dist/pypy/translator/c/src/thread.c Fri Sep 14 15:27:43 2007 @@ -1,2 +1,2 @@ #include -#include +#include "src/thread.h" From arigo at codespeak.net Fri Sep 14 15:30:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 15:30:51 +0200 (CEST) Subject: [pypy-svn] r46583 - pypy/dist/pypy/translator/tool Message-ID: <20070914133051.8951680CC@code0.codespeak.net> Author: arigo Date: Fri Sep 14 15:30:51 2007 New Revision: 46583 Modified: pypy/dist/pypy/translator/tool/cbuild.py Log: Ouch, it's more complicated than expected to force a recompile. Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Fri Sep 14 15:30:51 2007 @@ -128,7 +128,7 @@ libraries=libraries,) ], 'script_name': 'setup.py', - 'script_args': ['-q', 'build_ext', '--inplace'], + 'script_args': ['-q', 'build_ext', '--inplace', '--force'], } dist = Distribution(attrs) if not dist.parse_command_line(): @@ -165,13 +165,10 @@ cache_dir = py.path.local(pypydir).join('_cache') else: cache_dir = py.path.local(cache_dir) - if cache_dir.check(dir=1): - # XXX check timestamps of all cfiles - if 1: # not cache_dir.join(modname + '.so').check(): - modname = str(cache_dir.join(modname)) - compile_c_module(cfiles, modname, include_dirs=include_dirs, - libraries=libraries) - + assert cache_dir.check(dir=1) # XXX + modname = str(cache_dir.join(modname)) + compile_c_module(cfiles, modname, include_dirs=include_dirs, + libraries=libraries) def make_module_from_c(cfile, include_dirs=None, libraries=[]): cfile = py.path.local(cfile) From arigo at codespeak.net Fri Sep 14 19:21:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 19:21:42 +0200 (CEST) Subject: [pypy-svn] r46587 - in pypy/dist/pypy: interpreter module/_file module/_file/test module/thread rpython/lltypesystem Message-ID: <20070914172142.A14DC80E6@code0.codespeak.net> Author: arigo Date: Fri Sep 14 19:21:36 2007 New Revision: 46587 Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/module/_file/app_file.py pypy/dist/pypy/module/_file/interp_file.py pypy/dist/pypy/module/_file/test/test_file.py pypy/dist/pypy/module/thread/os_lock.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Proper locking for app-level files to protect the streamio from corruption in multithreaded usage. A pypy-c-thread now works correctly as long as we manually patch the C sources to make pypy_g_ExcData a gcc thread-local... Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Fri Sep 14 19:21:36 2007 @@ -434,6 +434,19 @@ from pypy.interpreter import pyframe return pyframe.PyFrame(self, code, w_globals, closure) + def allocate_lock(self): + """Return an interp-level Lock object if threads are enabled, + and a dummy object if they are not.""" + if self.config.objspace.usemodules.thread: + from pypy.module.thread.ll_thread import allocate_lock, error + try: + return allocate_lock() + except error: + raise OperationError(self.w_RuntimeError, + self.wrap("out of resources")) + else: + return dummy_lock + # Following is a friendly interface to common object space operations # that can be defined in term of more primitive ones. Subclasses # may also override specific functions for performance. @@ -859,6 +872,15 @@ space.exec_(source.compile(), w_glob, w_glob) return space.getitem(w_glob, space.wrap('anonymous')) +class DummyLock(object): + def acquire(self, flag): + return True + def release(self): + pass + def _freeze_(self): + return True +dummy_lock = DummyLock() + ## Table describing the regular part of the interface of object spaces, ## namely all methods which only take w_ arguments and return a w_ result ## (if any). Note: keep in sync with pypy.objspace.flow.operation.Table. Modified: pypy/dist/pypy/module/_file/app_file.py ============================================================================== --- pypy/dist/pypy/module/_file/app_file.py (original) +++ pypy/dist/pypy/module/_file/app_file.py Fri Sep 14 19:21:36 2007 @@ -102,12 +102,16 @@ return self.stream.readall() else: result = [] - while n > 0: - data = self.stream.read(n) - if not data: - break - n -= len(data) - result.append(data) + self.stream.lock() + try: + while n > 0: + data = self.stream.read(n) + if not data: + break + n -= len(data) + result.append(data) + finally: + self.stream.unlock() return ''.join(result) def readline(self, size=-1): @@ -125,20 +129,24 @@ else: # very inefficient unless there is a peek() result = [] - while size > 0: - # "peeks" on the underlying stream to see how many characters - # we can safely read without reading past an end-of-line - peeked = self.stream.peek() - pn = peeked.find("\n", 0, size) - if pn < 0: - pn = min(size-1, len(peeked)) - c = self.stream.read(pn + 1) - if not c: - break - result.append(c) - if c.endswith('\n'): - break - size -= len(c) + self.stream.lock() + try: + while size > 0: + # "peeks" on the underlying stream to see how many chars + # we can safely read without reading past an end-of-line + peeked = self.stream.peek() + pn = peeked.find("\n", 0, size) + if pn < 0: + pn = min(size-1, len(peeked)) + c = self.stream.read(pn + 1) + if not c: + break + result.append(c) + if c.endswith('\n'): + break + size -= len(c) + finally: + self.stream.unlock() return ''.join(result) def readlines(self, size=-1): @@ -151,17 +159,21 @@ raise ValueError('I/O operation on closed file') if not isinstance(size, (int, long)): raise TypeError("an integer is required") - if size < 0: - return list(iter(self.stream.readline, "")) - else: - result = [] - while size > 0: - line = self.stream.readline() - if not line: - break - result.append(line) - size -= len(line) - return result + self.stream.lock() + try: + if size < 0: + result = list(iter(self.stream.readline, "")) + else: + result = [] + while size > 0: + line = self.stream.readline() + if not line: + break + result.append(line) + size -= len(line) + finally: + self.stream.unlock() + return result def write(self, data): """write(str) -> None. Write string str to file. @@ -226,9 +238,13 @@ Size defaults to the current file position, as returned by tell().""" if self._closed: raise ValueError('I/O operation on closed file') - if size is None: - size = self.stream.tell() - self.stream.truncate(size) + self.stream.lock() + try: + if size is None: + size = self.stream.tell() + self.stream.truncate(size) + finally: + self.stream.unlock() def flush(self): """flush() -> None. Flush the internal I/O buffer.""" @@ -243,10 +259,17 @@ further I/O operations. close() may be called more than once without error. Some kinds of file objects (for example, opened by popen()) may return an exit status upon closing.""" + # use the stream lock to avoid double-closes or + # close-while-another-thread-uses-it. if not self._closed and hasattr(self, 'stream'): - self._closed = True - sys.pypy__exithandlers__.pop(self.stream, None) - self.stream.close() + self.stream.lock() + try: + if not self._closed: # could have changed... + self._closed = True + sys.pypy__exithandlers__.pop(self.stream, None) + self.stream.close() + finally: + self.stream.unlock() __del__ = close Modified: pypy/dist/pypy/module/_file/interp_file.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_file.py (original) +++ pypy/dist/pypy/module/_file/interp_file.py Fri Sep 14 19:21:36 2007 @@ -34,25 +34,83 @@ class W_Stream(Wrappable): + slock = None + slockowner = None + # Locking issues: + # * Multiple threads can access the same W_Stream in + # parallel, because many of the streamio calls eventually + # release the GIL in some external function call. + # * Parallel accesses have bad (and crashing) effects on the + # internal state of the buffering levels of the stream in + # particular. + # * We can't easily have a lock on each W_Stream because we + # can't translate prebuilt lock objects. + # We are still protected by the GIL, so the easiest is to create + # the lock on-demand. + def __init__(self, space, stream): + self.space = space self.stream = stream + def try_acquire_lock(self): + # this function runs with the GIL acquired so there is no race + # condition in the creation of the lock + if self.slock is None: + self.slock = self.space.allocate_lock() + me = self.space.getexecutioncontext() # used as thread ident + if self.slockowner is me: + return False # already acquired by the current thread + self.slock.acquire(True) + self.slockowner = me + return True + + def release_lock(self): + self.slockowner = None + self.slock.release() + + def descr_lock(self): + if not self.try_acquire_lock(): + raise OperationError(self.space.w_RuntimeError, + self.space.wrap("stream lock already held")) + + def descr_unlock(self): + me = self.space.getexecutioncontext() # used as thread ident + if self.slockowner is not me: + raise OperationError(self.space.w_RuntimeError, + self.space.wrap("stream lock is not held")) + self.release_lock() + + def _freeze_(self): + # remove the lock object, which will be created again as need at + # run-time. + self.slock = None + assert self.slockowner is None + return False + for name, argtypes in streamio.STREAM_METHODS.iteritems(): numargs = len(argtypes) args = ", ".join(["v%s" % i for i in range(numargs)]) exec py.code.Source(""" def %(name)s(self, space, %(args)s): + acquired = self.try_acquire_lock() try: - return space.wrap(self.stream.%(name)s(%(args)s)) - except streamio.StreamError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.message)) - except OSError, e: - raise wrap_oserror_as_ioerror(space, e) + try: + result = self.stream.%(name)s(%(args)s) + except streamio.StreamError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.message)) + except OSError, e: + raise wrap_oserror_as_ioerror(space, e) + finally: + if acquired: + self.release_lock() + return space.wrap(result) %(name)s.unwrap_spec = [W_Stream, ObjSpace] + argtypes """ % locals()).compile() in globals() W_Stream.typedef = TypeDef("Stream", + lock = interp2app(W_Stream.descr_lock), + unlock = interp2app(W_Stream.descr_unlock), **dict([(name, interp2app(globals()[name])) for name, _ in streamio.STREAM_METHODS.iteritems()])) Modified: pypy/dist/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/dist/pypy/module/_file/test/test_file.py (original) +++ pypy/dist/pypy/module/_file/test/test_file.py Fri Sep 14 19:21:36 2007 @@ -11,10 +11,8 @@ def test_simple(self): import _file f = _file.file(self.temppath, "w") - try: - f.write("foo") - finally: - f.close() + f.write("foo") + f.close() f = _file.file(self.temppath, "r") raises(TypeError, f.read, None) try: @@ -117,6 +115,39 @@ assert type(res) is str f.close() +class AppTestConcurrency(object): + # these tests only really make sense on top of a translated pypy-c, + # because on top of py.py the inner calls to os.write() don't + # release our object space's GIL. + def setup_class(cls): + cls.space = gettestobjspace(usemodules=("_file", "thread")) + cls.w_temppath = cls.space.wrap( + str(py.test.ensuretemp("fileimpl").join("concurrency.txt"))) + + def test_concurrent_writes(self): + # check that f.write() is atomic + import thread, _file, time + f = _file.file(self.temppath, "w+b") + def writer(i): + for j in range(150): + f.write('%3d %3d\n' % (i, j)) + locks[i].release() + locks = [] + for i in range(10): + lock = thread.allocate_lock() + lock.acquire() + locks.append(lock) + for i in range(10): + thread.start_new_thread(writer, (i,)) + # wait until all threads are done + for i in range(10): + locks[i].acquire() + f.seek(0) + lines = f.readlines() + lines.sort() + assert lines == ['%3d %3d\n' % (i, j) for i in range(10) + for j in range(150)] + f.close() def test_flush_at_exit(): from pypy import conftest Modified: pypy/dist/pypy/module/thread/os_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/os_lock.py (original) +++ pypy/dist/pypy/module/thread/os_lock.py Fri Sep 14 19:21:36 2007 @@ -30,6 +30,7 @@ "A wrappable box around an interp-level lock object." def __init__(self): + # XXX catch thread.error! self.lock = thread.allocate_lock() def descr_lock_acquire(self, space, waitflag=1): Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Fri Sep 14 19:21:36 2007 @@ -86,9 +86,11 @@ before = aroundstate.before after = aroundstate.after if before: before() - result = funcptr(*real_args) - if invoke_around_handlers: - if after: after() + try: + result = funcptr(*real_args) + finally: + if invoke_around_handlers: + if after: after() if stringpolicy == 'fullauto': for i, tp in unrolling_arg_tps: if tp is CCHARP: From arigo at codespeak.net Fri Sep 14 19:50:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 19:50:40 +0200 (CEST) Subject: [pypy-svn] r46588 - in pypy/dist/pypy: module/thread rlib rpython/lltypesystem Message-ID: <20070914175040.2C52180E5@code0.codespeak.net> Author: arigo Date: Fri Sep 14 19:50:38 2007 New Revision: 46588 Modified: pypy/dist/pypy/module/thread/gil.py pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Trying to convince the exception transformer to not put a check at the end of the call to before(). Modified: pypy/dist/pypy/module/thread/gil.py ============================================================================== --- pypy/dist/pypy/module/thread/gil.py (original) +++ pypy/dist/pypy/module/thread/gil.py Fri Sep 14 19:50:38 2007 @@ -80,6 +80,8 @@ spacestate = SpaceState() def before_external_call(): + # this function must not raise, in such a way that the exception + # transformer knows that it cannot raise! spacestate.GIL.release() def after_external_call(): Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Fri Sep 14 19:50:38 2007 @@ -155,9 +155,13 @@ # NOTE: the hooks are cleared during translation! To be effective # in a compiled program they must be set at run-time. from pypy.rpython.lltypesystem import rffi + rffi.aroundstate.before = before + rffi.aroundstate.after = after + # the 'aroundstate' contains regular function and not ll pointers to them, + # but let's call llhelper() anyway to force their annotation from pypy.rpython.annlowlevel import llhelper - rffi.aroundstate.before = llhelper(rffi.AroundFnPtr, before) - rffi.aroundstate.after = llhelper(rffi.AroundFnPtr, after) + llhelper(rffi.AroundFnPtr, before) + llhelper(rffi.AroundFnPtr, after) class UnboxedValue(object): Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Fri Sep 14 19:50:38 2007 @@ -86,11 +86,11 @@ before = aroundstate.before after = aroundstate.after if before: before() - try: - result = funcptr(*real_args) - finally: - if invoke_around_handlers: - if after: after() + # NB. it is essential that no exception checking occurs after + # the call to before(), because we don't have the GIL any more! + result = funcptr(*real_args) + if invoke_around_handlers: + if after: after() if stringpolicy == 'fullauto': for i, tp in unrolling_arg_tps: if tp is CCHARP: @@ -104,8 +104,8 @@ AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) class AroundState: def _freeze_(self): - self.before = lltype.nullptr(AroundFnPtr.TO) - self.after = lltype.nullptr(AroundFnPtr.TO) + self.before = None # or a regular RPython function + self.after = None # or a regular RPython function return False aroundstate = AroundState() aroundstate._freeze_() From arigo at codespeak.net Fri Sep 14 20:29:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 20:29:04 +0200 (CEST) Subject: [pypy-svn] r46589 - in pypy/dist/pypy/module/thread: . test Message-ID: <20070914182904.0C3F780F1@code0.codespeak.net> Author: arigo Date: Fri Sep 14 20:29:04 2007 New Revision: 46589 Added: pypy/dist/pypy/module/thread/error.py (contents, props changed) Modified: pypy/dist/pypy/module/thread/gil.py pypy/dist/pypy/module/thread/ll_thread.py pypy/dist/pypy/module/thread/os_lock.py pypy/dist/pypy/module/thread/os_thread.py pypy/dist/pypy/module/thread/test/test_thread.py Log: Fix error handling (at least, attempt to). Note that there is a race condition somewhere on top of py.py making test_local crash but very rarely :-(( Added: pypy/dist/pypy/module/thread/error.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/thread/error.py Fri Sep 14 20:29:04 2007 @@ -0,0 +1,6 @@ +from pypy.interpreter.error import OperationError + +def reraise_thread_error(space, msg): + w_module = space.getbuiltinmodule('thread') + w_error = space.getattr(w_module, space.wrap('error')) + raise OperationError(w_error, space.wrap(msg)) Modified: pypy/dist/pypy/module/thread/gil.py ============================================================================== --- pypy/dist/pypy/module/thread/gil.py (original) +++ pypy/dist/pypy/module/thread/gil.py Fri Sep 14 20:29:04 2007 @@ -8,6 +8,7 @@ # from time to time, using the executioncontext's XXX from pypy.module.thread import ll_thread as thread +from pypy.module.thread.error import reraise_thread_error from pypy.interpreter.miscutils import Action from pypy.module.thread.threadlocals import OSThreadLocals from pypy.rlib.objectmodel import invoke_around_extcall @@ -19,7 +20,10 @@ def setup_threads(self, space): """Enable threads in the object space, if they haven't already been.""" if self.GIL is None: - self.GIL = thread.allocate_lock_NOAUTO() + try: + self.GIL = thread.allocate_lock_NOAUTO() + except thread.error: + reraise_thread_error(space, "can't allocate GIL") self.enter_thread(space) # setup the main thread # add the GIL-releasing callback as an action on the space space.pending_actions.append(GILReleaseAction(self)) Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Fri Sep 14 20:29:04 2007 @@ -11,11 +11,8 @@ from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.lltype import typeOf from pypy.rlib.objectmodel import debug_assert -from pypy.rlib.nonconst import NonConstant -class error(Exception): - def __init__(self, msg): - self.msg = msg +error = thread.error from pypy.tool.autopath import pypydir pypydir = py.path.local(pypydir) @@ -138,7 +135,7 @@ # Sanity check: the lock must be locked if self.acquire(False): c_thread_releaselock(self._lock) - raise error(NonConstant("bad lock")) + raise error("bad lock") else: c_thread_releaselock(self._lock) Modified: pypy/dist/pypy/module/thread/os_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/os_lock.py (original) +++ pypy/dist/pypy/module/thread/os_lock.py Fri Sep 14 20:29:04 2007 @@ -3,7 +3,7 @@ """ from pypy.module.thread import ll_thread as thread -from pypy.interpreter.error import OperationError +from pypy.module.thread.error import reraise_thread_error from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import ObjSpace, interp2app from pypy.interpreter.typedef import TypeDef @@ -29,9 +29,11 @@ class Lock(Wrappable): "A wrappable box around an interp-level lock object." - def __init__(self): - # XXX catch thread.error! - self.lock = thread.allocate_lock() + def __init__(self, space): + try: + self.lock = thread.allocate_lock() + except thread.error: + reraise_thread_error(space, "out of resources") def descr_lock_acquire(self, space, waitflag=1): """Lock the lock. Without argument, this blocks if the lock is already @@ -51,9 +53,7 @@ try: self.lock.release() except thread.error: - w_module = space.getbuiltinmodule('thread') - w_error = space.getattr(w_module, space.wrap('error')) - raise OperationError(w_error, space.wrap("release unlocked lock")) + reraise_thread_error(space, "release unlocked lock") def descr_lock_locked(self, space): """Return whether the lock is in the locked state.""" @@ -96,7 +96,7 @@ def allocate_lock(space): """Create a new lock object. (allocate() is an obsolete synonym.) See LockType.__doc__ for information about locks.""" - return space.wrap(Lock()) + return space.wrap(Lock(space)) def getlocktype(space): return space.gettypeobject(Lock.typedef) Modified: pypy/dist/pypy/module/thread/os_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/os_thread.py (original) +++ pypy/dist/pypy/module/thread/os_thread.py Fri Sep 14 20:29:04 2007 @@ -3,6 +3,7 @@ """ from pypy.module.thread import ll_thread as thread +from pypy.module.thread.error import reraise_thread_error from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments @@ -81,7 +82,10 @@ boot.space = space boot.w_callable = w_callable boot.args = args - ident = thread.start_new_thread(Bootstrapper.bootstrap, (boot,)) + try: + ident = thread.start_new_thread(Bootstrapper.bootstrap, (boot,)) + except thread.error: + reraise_thread_error(space, "can't start new thread") return space.wrap(ident) Modified: pypy/dist/pypy/module/thread/test/test_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_thread.py Fri Sep 14 20:29:04 2007 @@ -1,7 +1,31 @@ +import thread, time from pypy.module.thread.test.support import GenericTestThread class AppTestThread(GenericTestThread): + def setup_class(cls): + GenericTestThread.setup_class.im_func(cls) + # if we cannot start more than, say, 1000 threads on this OS, then + # we can check that we get the proper error at app-level + space = cls.space + lock = thread.allocate_lock() + lock.acquire() + def f(): + lock.acquire() + lock.release() + try: + try: + for i in range(1000): + thread.start_new_thread(f, ()) + finally: + lock.release() + # wait a bit to allow most threads to finish now + time.sleep(0.5) + except (thread.error, MemoryError): + cls.w_can_start_many_threads = space.w_False + else: + cls.w_can_start_many_threads = space.w_True + def test_start_new_thread(self): import thread feedback = [] @@ -141,3 +165,25 @@ self.waitfor(lambda: done, delay=3) assert done # see stderr for failures in threads assert sorted(lst) == range(120) + + def test_many_threads(self): + import thread, time + if self.can_start_many_threads: + skip("this OS supports too many threads to check (> 1000)") + lock = thread.allocate_lock() + lock.acquire() + def f(): + lock.acquire() + lock.release() + try: + try: + for i in range(1000): + thread.start_new_thread(f, ()) + finally: + lock.release() + # wait a bit to allow most threads to finish now + self.busywait(2.0) + except (thread.error, MemoryError): + pass + else: + raise Exception("could unexpectedly start 1000 threads") From arigo at codespeak.net Fri Sep 14 20:56:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 20:56:47 +0200 (CEST) Subject: [pypy-svn] r46590 - pypy/dist/pypy/module/thread/test Message-ID: <20070914185647.3D4B880DD@code0.codespeak.net> Author: arigo Date: Fri Sep 14 20:56:44 2007 New Revision: 46590 Modified: pypy/dist/pypy/module/thread/test/test_thread.py Log: A very mean test. It shows various issues when run in various ways. Modified: pypy/dist/pypy/module/thread/test/test_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_thread.py Fri Sep 14 20:56:44 2007 @@ -22,9 +22,9 @@ # wait a bit to allow most threads to finish now time.sleep(0.5) except (thread.error, MemoryError): - cls.w_can_start_many_threads = space.w_False + cls.w_can_start_many_threads = space.wrap(False) else: - cls.w_can_start_many_threads = space.w_True + cls.w_can_start_many_threads = space.wrap(True) def test_start_new_thread(self): import thread @@ -187,3 +187,48 @@ pass else: raise Exception("could unexpectedly start 1000 threads") + + def test_parallel_writes_and_reads(self): + # this test seems to be very bad: + # * when run normally, there is an early deadlock + # * when run in plain CPython (py.test -A) the print >> fwrite + # eventually deadlocks - that looks like a CPython bug + # * when run as pypy-c py.test -A, I get a Fatal RPython error, + # about an RPython-level thread.error + skip("to be looked at more closely") + + import thread, os + read_fd, write_fd = os.pipe() + fread = os.fdopen(read_fd, 'rb', 200) + fwrite = os.fdopen(write_fd, 'wb', 200) + run = True + readers_done = 0 + + def writer(): + f = 0.1 + while run: + print >> fwrite, f, + f = 4*f - 3*f*f + print >> fwrite, "X" + + def reader(): + while True: + data = fread.read(1) + if data == "X": + break + readers_done += 1 + + for j in range(3): + thread.start_new_thread(reader, ()) + thread.start_new_thread(writer, ()) + + import time + t = time.time() + 5 + print "start of test" + while time.time() < t: + time.sleep(1) + print "end of test" + + assert readers_done == 0 + run = False # end the writers + self.waitfor(lambda: readers_done == 3) From arigo at codespeak.net Fri Sep 14 21:25:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Sep 2007 21:25:03 +0200 (CEST) Subject: [pypy-svn] r46591 - in pypy/dist/pypy/module: _file/test thread/test Message-ID: <20070914192503.C4A3780DD@code0.codespeak.net> Author: arigo Date: Fri Sep 14 21:25:02 2007 New Revision: 46591 Modified: pypy/dist/pypy/module/_file/test/test_file.py pypy/dist/pypy/module/thread/test/test_thread.py Log: Move this test into module/_file to avoid using CPython's own 'file' instead of explicitly our own. Modified: pypy/dist/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/dist/pypy/module/_file/test/test_file.py (original) +++ pypy/dist/pypy/module/_file/test/test_file.py Fri Sep 14 21:25:02 2007 @@ -149,6 +149,55 @@ for j in range(150)] f.close() + def test_parallel_writes_and_reads(self): + # this test seems to be very bad: + # * when run normally, there is an early deadlock + # * when run in plain CPython (py.test -A) the print >> fwrite + # eventually deadlocks - that looks like a CPython bug + # * when run as pypy-c py.test -A, I get a Fatal RPython error, + # about an RPython-level thread.error + skip("to be looked at more closely") + + import thread, os, _file + read_fd, write_fd = os.pipe() + fread = _file.file.fdopen(read_fd, 'rb', 200) + fwrite = _file.file.fdopen(write_fd, 'wb', 200) + run = True + readers_done = [0] + + def writer(): + f = 0.1 + while run: + print >> fwrite, f, + f = 4*f - 3*f*f + print >> fwrite, "X" + + def reader(): + while True: + data = fread.read(1) + if data == "X": + break + readers_done[0] += 1 + + for j in range(3): + thread.start_new_thread(reader, ()) + thread.start_new_thread(writer, ()) + + import time + t = time.time() + 5 + print "start of test" + while time.time() < t: + time.sleep(1) + print "end of test" + + assert readers_done[0] == 0 + run = False # end the writers + while readers_done[0] != 3: + print 'readers_done == %d, still waiting...' % (readers_done[0],) + time.sleep(0.4) + print 'Passed.' + + def test_flush_at_exit(): from pypy import conftest from pypy.tool.option import make_config, make_objspace Modified: pypy/dist/pypy/module/thread/test/test_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_thread.py Fri Sep 14 21:25:02 2007 @@ -187,48 +187,3 @@ pass else: raise Exception("could unexpectedly start 1000 threads") - - def test_parallel_writes_and_reads(self): - # this test seems to be very bad: - # * when run normally, there is an early deadlock - # * when run in plain CPython (py.test -A) the print >> fwrite - # eventually deadlocks - that looks like a CPython bug - # * when run as pypy-c py.test -A, I get a Fatal RPython error, - # about an RPython-level thread.error - skip("to be looked at more closely") - - import thread, os - read_fd, write_fd = os.pipe() - fread = os.fdopen(read_fd, 'rb', 200) - fwrite = os.fdopen(write_fd, 'wb', 200) - run = True - readers_done = 0 - - def writer(): - f = 0.1 - while run: - print >> fwrite, f, - f = 4*f - 3*f*f - print >> fwrite, "X" - - def reader(): - while True: - data = fread.read(1) - if data == "X": - break - readers_done += 1 - - for j in range(3): - thread.start_new_thread(reader, ()) - thread.start_new_thread(writer, ()) - - import time - t = time.time() + 5 - print "start of test" - while time.time() < t: - time.sleep(1) - print "end of test" - - assert readers_done == 0 - run = False # end the writers - self.waitfor(lambda: readers_done == 3) From simonb at codespeak.net Fri Sep 14 23:44:56 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Fri, 14 Sep 2007 23:44:56 +0200 (CEST) Subject: [pypy-svn] r46593 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070914214456.C510F80C9@code0.codespeak.net> Author: simonb Date: Fri Sep 14 23:44:45 2007 New Revision: 46593 Modified: pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: add dtype attribute, annotation of copy method Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Fri Sep 14 23:44:45 2007 @@ -46,7 +46,7 @@ def __init__(self, typecode, ndim=1): if not typecode in self.typecode_to_item: raise AnnotatorError("bad typecode: %r"%typecode) - self.typecode = typecode + self.dtype = self.typecode = typecode self.ndim = ndim def get_one_dim(self): @@ -66,6 +66,8 @@ s = SomeTuple([SomeInteger()]*s_array.ndim) elif attr == 'ndim': s = SomeInteger() + elif attr == 'dtype': + s = SomeChar() if s is None: return SomeObject.getattr(s_array, s_attr) return s @@ -81,6 +83,7 @@ def method_transpose(self): return SomeArray(self.typecode, self.ndim) + method_copy = method_transpose def method_astype(self, s_dtype): if isinstance(s_dtype, SomeChar) and s_dtype.is_constant(): Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Fri Sep 14 23:44:45 2007 @@ -17,7 +17,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rtuple from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rpython.lltypesystem.lltype import \ - GcArray, GcStruct, Number, Primitive, Signed, Ptr, Unsigned, Void, FixedSizeArray, Bool,\ + GcArray, GcStruct, Number, Primitive, Signed, Ptr, Unsigned, Char, Void, FixedSizeArray, Bool,\ GcForwardReference, malloc, direct_arrayitems, direct_ptradd, nullptr, typeMethod,\ cast_primitive from pypy.rpython.lltypesystem.rffi import cast @@ -315,6 +315,10 @@ cname = inputconst(Void, 'ndim') return hop.llops.genop('getfield', [v_array, cname], resulttype=Signed) + def get_dtype(self, hop, v_array): + cdtype = inputconst(Char, self.s_array.dtype) + return cdtype + def get_shape(self, hop, v_array): TUPLE = TUPLE_TYPE([Signed]*self.ndim) cARRAY = inputconst(lltype.Void, self.lowleveltype.TO) Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Fri Sep 14 23:44:45 2007 @@ -5,7 +5,7 @@ import py import pypy.rpython.numpy.implementation from pypy.annotation import model as annmodel -from pypy.annotation.model import SomeObject, SomeTuple +from pypy.annotation.model import SomeObject, SomeInteger, SomeChar, SomeTuple from pypy.annotation.annrpython import RPythonAnnotator from pypy.tool.error import AnnotatorError from pypy.translator.translator import TranslationContext, graphof @@ -78,6 +78,34 @@ s = a.build_types(access_with_variable, []) assert s.knowntype == rffi.r_int + def test_annotate_attr(self): + def f(): + a = numpy.empty((3,4,5)) + return a.ndim + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert isinstance(s, SomeInteger) + + def f(): + a = numpy.empty((3,4,5)) + return a.shape + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert isinstance(s, SomeTuple) + + def f(): + a = numpy.empty((3,4,5)) + return a.dtype + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(f, []) + assert isinstance(s, SomeChar) + def test_annotate_empty(self): def f(): a = numpy.empty((3,4,5)) @@ -240,6 +268,15 @@ assert s.ndim == 1 assert s.typecode == 'd' + def f_copy(): + a = numpy.array(range(12)) + return a.copy() + + s = a.build_types(f_copy, []) + assert type(s) == SomeArray + assert s.ndim == 1 + assert s.typecode == 'l' + def test_annotate_indexing(self): def f(): a = numpy.empty((4,3), dtype='i') @@ -316,6 +353,28 @@ assert res.dataptr[i] == 0 assert res.ndim == 2 + def test_specialize_array_attr(self): + def f(): + a = numpy.empty((3,4,5)) + return a.ndim + res = interpret(f, []) + assert res == 3 + + def f(): + a = numpy.empty((3,4,5)) + return a.shape + res = interpret(f, []) + assert res.item0 == 3 + assert res.item1 == 4 + assert res.item2 == 5 + + def f(): + a = numpy.empty((3,4,5)) + return a.dtype + res = interpret(f, []) + assert res == 'd' + + def test_specialize_array_access(self): def access_with_variable(): my_array = numpy.array(range(10), dtype='i') @@ -329,14 +388,6 @@ res = interpret(access_with_variable, []) assert res == 45 - def test_specialize_array_attr(self): - def f(): - a = numpy.array([1,2]) - return a.ndim - - res = interpret(f, []) - assert res == 1 - def test_specialize_array_attr_shape(self): def f(): a = numpy.empty((2,3)) @@ -392,6 +443,16 @@ assert res.dataptr[0] == 0. assert res.dataptr[1] == 0.5 + def f_copy(): + a = numpy.array(range(4)) + b = a.copy() + a[:] = 0 + return b + + res = interpret(f_copy, []) + for i in range(4): + assert res.dataptr[i] == i + def test_specialize_view_0(self): def f(): a = numpy.empty((4,3), dtype='i') From arigo at codespeak.net Sat Sep 15 10:30:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 10:30:13 +0200 (CEST) Subject: [pypy-svn] r46609 - pypy/dist/pypy/module/thread Message-ID: <20070915083013.E66F380C6@code0.codespeak.net> Author: arigo Date: Sat Sep 15 10:30:13 2007 New Revision: 46609 Modified: pypy/dist/pypy/module/thread/error.py pypy/dist/pypy/module/thread/gil.py pypy/dist/pypy/module/thread/os_lock.py pypy/dist/pypy/module/thread/os_thread.py Log: Make the flow space happy. Modified: pypy/dist/pypy/module/thread/error.py ============================================================================== --- pypy/dist/pypy/module/thread/error.py (original) +++ pypy/dist/pypy/module/thread/error.py Sat Sep 15 10:30:13 2007 @@ -1,6 +1,6 @@ from pypy.interpreter.error import OperationError -def reraise_thread_error(space, msg): +def wrap_thread_error(space, msg): w_module = space.getbuiltinmodule('thread') w_error = space.getattr(w_module, space.wrap('error')) - raise OperationError(w_error, space.wrap(msg)) + return OperationError(w_error, space.wrap(msg)) Modified: pypy/dist/pypy/module/thread/gil.py ============================================================================== --- pypy/dist/pypy/module/thread/gil.py (original) +++ pypy/dist/pypy/module/thread/gil.py Sat Sep 15 10:30:13 2007 @@ -8,7 +8,7 @@ # from time to time, using the executioncontext's XXX from pypy.module.thread import ll_thread as thread -from pypy.module.thread.error import reraise_thread_error +from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.miscutils import Action from pypy.module.thread.threadlocals import OSThreadLocals from pypy.rlib.objectmodel import invoke_around_extcall @@ -23,7 +23,7 @@ try: self.GIL = thread.allocate_lock_NOAUTO() except thread.error: - reraise_thread_error(space, "can't allocate GIL") + raise wrap_thread_error(space, "can't allocate GIL") self.enter_thread(space) # setup the main thread # add the GIL-releasing callback as an action on the space space.pending_actions.append(GILReleaseAction(self)) Modified: pypy/dist/pypy/module/thread/os_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/os_lock.py (original) +++ pypy/dist/pypy/module/thread/os_lock.py Sat Sep 15 10:30:13 2007 @@ -3,7 +3,7 @@ """ from pypy.module.thread import ll_thread as thread -from pypy.module.thread.error import reraise_thread_error +from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import ObjSpace, interp2app from pypy.interpreter.typedef import TypeDef @@ -33,7 +33,7 @@ try: self.lock = thread.allocate_lock() except thread.error: - reraise_thread_error(space, "out of resources") + raise wrap_thread_error(space, "out of resources") def descr_lock_acquire(self, space, waitflag=1): """Lock the lock. Without argument, this blocks if the lock is already @@ -53,7 +53,7 @@ try: self.lock.release() except thread.error: - reraise_thread_error(space, "release unlocked lock") + raise wrap_thread_error(space, "release unlocked lock") def descr_lock_locked(self, space): """Return whether the lock is in the locked state.""" Modified: pypy/dist/pypy/module/thread/os_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/os_thread.py (original) +++ pypy/dist/pypy/module/thread/os_thread.py Sat Sep 15 10:30:13 2007 @@ -3,7 +3,7 @@ """ from pypy.module.thread import ll_thread as thread -from pypy.module.thread.error import reraise_thread_error +from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments @@ -85,7 +85,7 @@ try: ident = thread.start_new_thread(Bootstrapper.bootstrap, (boot,)) except thread.error: - reraise_thread_error(space, "can't start new thread") + raise wrap_thread_error(space, "can't start new thread") return space.wrap(ident) From arigo at codespeak.net Sat Sep 15 10:37:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 10:37:59 +0200 (CEST) Subject: [pypy-svn] r46610 - pypy/dist/pypy/translator/tool/test Message-ID: <20070915083759.9E81F80C9@code0.codespeak.net> Author: arigo Date: Sat Sep 15 10:37:59 2007 New Revision: 46610 Modified: pypy/dist/pypy/translator/tool/test/test_cbuild.py Log: Ha, this test was never run because of the agressive caching. Modified: pypy/dist/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/dist/pypy/translator/tool/test/test_cbuild.py Sat Sep 15 10:37:59 2007 @@ -28,7 +28,7 @@ from pypy.tool.autopath import pypydir pypydir = py.path.local(pypydir) csourcedir = pypydir.join('translator', 'c', 'src') - include_dirs = [str(csourcedir)] + include_dirs = [str(csourcedir.dirpath())] files = [csourcedir.join('thread.c')] cache_c_module(files, '_thread', cache_dir=udir, include_dirs=include_dirs, libraries=['pthread']) From arigo at codespeak.net Sat Sep 15 10:49:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 10:49:51 +0200 (CEST) Subject: [pypy-svn] r46611 - pypy/dist/pypy/translator/c Message-ID: <20070915084951.0F9BA80C4@code0.codespeak.net> Author: arigo Date: Sat Sep 15 10:49:45 2007 New Revision: 46611 Modified: pypy/dist/pypy/translator/c/genc.py Log: Fix the generated setup.py. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Sat Sep 15 10:49:45 2007 @@ -144,7 +144,8 @@ cfile, extra, extraincludes = gen_source(db, modulename, targetdir, defines = defines, exports = self.exports, - symboltable = self.symboltable) + symboltable = self.symboltable, + libraries = self.libraries) else: if self.config.translation.instrument: defines['INSTRUMENT'] = 1 @@ -704,7 +705,7 @@ def gen_source(database, modulename, targetdir, defines={}, exports={}, - symboltable=None): + symboltable=None, libraries=[]): assert not database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -881,7 +882,8 @@ ext_modules = [Extension(name = "%(modulename)s", sources = ["%(modulename)s.c"], extra_compile_args = extra_compile_args, - include_dirs = [PYPY_INCLUDE_DIR])]) + include_dirs = [PYPY_INCLUDE_DIR], + libraries = %(libraries)r)]) ''' MAKEFILE = ''' From antocuni at codespeak.net Sat Sep 15 11:11:36 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 15 Sep 2007 11:11:36 +0200 (CEST) Subject: [pypy-svn] r46613 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070915091136.1614C80C7@code0.codespeak.net> Author: antocuni Date: Sat Sep 15 11:11:35 2007 New Revision: 46613 Modified: pypy/dist/pypy/translator/jvm/test/test_backendopt.py Log: skip these failing tests Modified: pypy/dist/pypy/translator/jvm/test/test_backendopt.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_backendopt.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_backendopt.py Sat Sep 15 11:11:35 2007 @@ -6,3 +6,9 @@ def getcompiled(self, fn, annotation): t = JvmTest() return t._compile(fn, None, annotation, backendopt=True) + + def test_longlong_switch(self): + py.test.skip('fixme!') + + def test_ulonglong_switch(self): + py.test.skip('fixme!') From arigo at codespeak.net Sat Sep 15 11:34:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 11:34:01 +0200 (CEST) Subject: [pypy-svn] r46615 - in pypy/dist/pypy: jit/codegen/i386 rpython rpython/lltypesystem Message-ID: <20070915093401.DF8A180C6@code0.codespeak.net> Author: arigo Date: Sat Sep 15 11:34:01 2007 New Revision: 46615 Modified: pypy/dist/pypy/jit/codegen/i386/codebuf.py pypy/dist/pypy/jit/codegen/i386/codebuf_nt.py pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/rbuiltin.py Log: Port codebuf from rctypes to rffi. This is prompted by the fact that rctypes generates C code that looks like it's doing out-of-bound array accesses when RPY_LL_ASSERT is enabled. Step 1: test_codebuf passes on Linux again. Modified: pypy/dist/pypy/jit/codegen/i386/codebuf.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/codebuf.py (original) +++ pypy/dist/pypy/jit/codegen/i386/codebuf.py Sat Sep 15 11:34:01 2007 @@ -1,5 +1,5 @@ import os -from ctypes import POINTER, cast, c_char, c_void_p, CFUNCTYPE, c_int +from pypy.rpython.lltypesystem import lltype, rffi from ri386 import I386CodeBuilder # ____________________________________________________________ @@ -19,8 +19,7 @@ def __init__(self, start, end): map_size = end - start - res = c_void_p(start) - data = cast(res, POINTER(c_char * map_size)) + data = rffi.cast(PTR, start) self._init(data, map_size) def _init(self, data, map_size): @@ -33,12 +32,12 @@ if p + len(data) > self._size: raise CodeBlockOverflow for c in data: - self._data.contents[p] = c + self._data[p] = c p += 1 self._pos = p def tell(self): - baseaddr = cast(self._data, c_void_p).value + baseaddr = rffi.cast(lltype.Signed, self._data) return baseaddr + self._pos def seekback(self, count): @@ -48,7 +47,7 @@ def execute(self, arg1, arg2): # XXX old testing stuff - fnptr = cast(self._data, binaryfn) + fnptr = rffi.cast(lltype.Ptr(BINARYFN), self._data) return fnptr(arg1, arg2) def done(self): @@ -63,6 +62,9 @@ machine_code_dumper.dump(self, 'LOG', self._pos, msg) +BINARYFN = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + + class MachineCodeDumper: enabled = True log_fd = -1 @@ -107,7 +109,7 @@ HEX = '0123456789ABCDEF' dump = [] for p in range(start, end): - o = ord(cb._data.contents[p]) + o = ord(cb._data[p]) dump.append(HEX[o >> 4]) dump.append(HEX[o & 15]) if (p & 3) == 3: @@ -120,14 +122,11 @@ class MachineCodeBlock(InMemoryCodeBuilder): def __init__(self, map_size): - res = memhandler.alloc(map_size) - data = cast(res, POINTER(c_char * map_size)) + data = memhandler.alloc(map_size) self._init(data, map_size) def __del__(self): - memhandler.free(cast(self._data, PTR), self._size) - -binaryfn = CFUNCTYPE(c_int, c_int, c_int) # for testing + memhandler.free(self._data, self._size) # ____________________________________________________________ Modified: pypy/dist/pypy/jit/codegen/i386/codebuf_nt.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/codebuf_nt.py (original) +++ pypy/dist/pypy/jit/codegen/i386/codebuf_nt.py Sat Sep 15 11:34:01 2007 @@ -5,6 +5,9 @@ from pypy.rpython.rctypes.tool.ctypes_platform import SimpleType +raise NotImplementedError("this needs to be ported from rctypes to rffi") + + class CConfig: _header_ = '#include ' Modified: pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py (original) +++ pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py Sat Sep 15 11:34:01 2007 @@ -1,21 +1,19 @@ -import pypy.rpython.rctypes.implementation # register rctypes types -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc -from ctypes import POINTER, c_char, c_int, cast, c_void_p +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.tool import rffi_platform as platform class CConfig: _includes_ = ("sys/types.h", "sys/mman.h") - size_t = ctypes_platform.SimpleType("size_t", c_int) - off_t = ctypes_platform.SimpleType("off_t", c_int) + size_t = platform.SimpleType("size_t", rffi.ULONG) + off_t = platform.SimpleType("off_t", rffi.LONG) - MAP_PRIVATE = ctypes_platform.DefinedConstantInteger("MAP_PRIVATE") - MAP_ANON = ctypes_platform.DefinedConstantInteger("MAP_ANON") - MAP_ANONYMOUS = ctypes_platform.DefinedConstantInteger("MAP_ANONYMOUS") - PROT_READ = ctypes_platform.DefinedConstantInteger("PROT_READ") - PROT_WRITE = ctypes_platform.DefinedConstantInteger("PROT_WRITE") - PROT_EXEC = ctypes_platform.DefinedConstantInteger("PROT_EXEC") + MAP_PRIVATE = platform.ConstantInteger("MAP_PRIVATE") + MAP_ANON = platform.DefinedConstantInteger("MAP_ANON") + MAP_ANONYMOUS = platform.DefinedConstantInteger("MAP_ANONYMOUS") + PROT_READ = platform.ConstantInteger("PROT_READ") + PROT_WRITE = platform.ConstantInteger("PROT_WRITE") + PROT_EXEC = platform.ConstantInteger("PROT_EXEC") -globals().update(ctypes_platform.configure(CConfig)) +globals().update(platform.configure(CConfig)) if MAP_ANONYMOUS is None: MAP_ANONYMOUS = MAP_ANON assert MAP_ANONYMOUS is not None @@ -23,16 +21,16 @@ # ____________________________________________________________ -PTR = POINTER(c_char) # cannot use c_void_p as return value of functions :-( +PTR = rffi.CCHARP -mmap_ = libc.mmap -mmap_.argtypes = [PTR, size_t, c_int, c_int, c_int, off_t] -mmap_.restype = PTR -mmap_.includes = ("sys/mman.h",) -munmap_ = libc.munmap -munmap_.argtypes = [PTR, size_t] -munmap_.restype = c_int -munmap_.includes = ("sys/mman.h",) +mmap_ = rffi.llexternal('mmap', + [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], + PTR, + includes = ["sys/mman.h"]) +munmap_ = rffi.llexternal('munmap', + [PTR, size_t], + rffi.INT, + includes = ["sys/mman.h"]) class Hint: pos = -0x4fff0000 # for reproducible results @@ -41,8 +39,9 @@ def alloc(map_size): flags = MAP_PRIVATE | MAP_ANONYMOUS prot = PROT_EXEC | PROT_READ | PROT_WRITE - res = mmap_(cast(c_void_p(hint.pos), PTR), map_size, prot, flags, -1, 0) - if cast(res, c_void_p).value == -1: + hintp = rffi.cast(PTR, hint.pos) + res = mmap_(hintp, map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): raise MemoryError hint.pos += map_size return res Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Sat Sep 15 11:34:01 2007 @@ -515,7 +515,7 @@ cptr = ctypes.cast(cvalue, cresulttype) return ctypes2lltype(RESTYPE, cptr) # first cast the input pointer to an integer - cvalue = ctypes.c_void_p(cvalue).value + cvalue = ctypes.cast(cvalue, ctypes.c_void_p).value elif isinstance(cvalue, (str, unicode)): cvalue = ord(cvalue) # character -> integer Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Sat Sep 15 11:34:01 2007 @@ -452,6 +452,11 @@ return llops.genop('cast_pointer', [v_value], resulttype = TGT) elif ORIG == llmemory.Address: return llops.genop('cast_adr_to_ptr', [v_value], resulttype = TGT) + elif isinstance(ORIG, lltype.Primitive) and ORIG in _cast_to_Signed: + op = _cast_to_Signed[ORIG] + if op: + v_value = llops.genop(op, [v_value], resulttype=lltype.Signed) + return llops.genop('cast_int_to_ptr', [v_value], resulttype=TGT) elif TGT == llmemory.Address and isinstance(ORIG, lltype.Ptr): return llops.genop('cast_ptr_to_adr', [v_value], resulttype = TGT) raise TypeError("don't know how to cast from %r to %r" % (ORIG, TGT)) From antocuni at codespeak.net Sat Sep 15 11:35:03 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 15 Sep 2007 11:35:03 +0200 (CEST) Subject: [pypy-svn] r46616 - in pypy/dist/pypy/translator: c/test cli jvm oosupport Message-ID: <20070915093503.E7C4B80C6@code0.codespeak.net> Author: antocuni Date: Sat Sep 15 11:35:03 2007 New Revision: 46616 Modified: pypy/dist/pypy/translator/c/test/test_backendoptimized.py pypy/dist/pypy/translator/cli/function.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/oosupport/function.py Log: use jvm's tableswitch to implement numeric switch. Modified: pypy/dist/pypy/translator/c/test/test_backendoptimized.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_backendoptimized.py (original) +++ pypy/dist/pypy/translator/c/test/test_backendoptimized.py Sat Sep 15 11:35:03 2007 @@ -104,6 +104,34 @@ for x in (0,1,2,3,9,27,48, -9): assert fn(x) == f(x) + def test_int_switch_nonsparse(self): + def f(x): + if x == 1: + return 9 + elif x == 2: + return 27 + elif x == 3: + return 3 + return 0 + codegenerator = self.CodeGenerator() + fn = codegenerator.getcompiled(f, [int]) + for x in (0,1,2,3,9,27,48, -9): + assert fn(x) == f(x) + + def test_int_switch_nonsparse_neg(self): + def f(x): + if x == -1: + return 9 + elif x == 2: + return 27 + elif x == 3: + return 3 + return 0 + codegenerator = self.CodeGenerator() + fn = codegenerator.getcompiled(f, [int]) + for x in (0,1,2,3,9,27,48, -9): + assert fn(x) == f(x) + def test_uint_switch(self): def f(x): if x == r_uint(3): Modified: pypy/dist/pypy/translator/cli/function.py ============================================================================== --- pypy/dist/pypy/translator/cli/function.py (original) +++ pypy/dist/pypy/translator/cli/function.py Sat Sep 15 11:35:03 2007 @@ -166,28 +166,10 @@ self.render_numeric_switch_naive(block) return - cases = {} - naive = False - for link in block.exits: - if link.exitcase == "default": - default = link, self.next_label('switch') - else: - if block.exitswitch.concretetype in (ootype.Char, ootype.UniChar): - value = ord(link.exitcase) - else: - value = link.exitcase - if value < 0: - naive = True - break - cases[value] = link, self.next_label('switch') - - try: - max_case = max(cases.keys()) - except ValueError: - max_case = 0 - if max_case > 3*len(cases) + 10: # the switch is very sparse, better to use the naive version - naive = True + cases, min_case, max_case, default = self._collect_switch_cases(block) + is_sparse = self._is_sparse_switch(cases, min_case, max_case) + naive = (min_case < 0) or is_sparse if naive: self.render_numeric_switch_naive(block) return @@ -202,12 +184,6 @@ for link, lbl in cases.itervalues(): self.render_switch_case(link, lbl) - def render_switch_case(self, link, label): - target_label = self._get_block_name(link.target) - self.set_label(label) - self._setup_link(link) - self.generator.branch_unconditionally(target_label) - # Those parts of the generator interface that are function # specific Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sat Sep 15 11:35:03 2007 @@ -1380,6 +1380,7 @@ return self.curfunc.instr_counter def emit_tableswitch(self, low, lbls, default): - pass - - + self.curclass.out(' tableswitch %d\n' % low) + for label in lbls: + self.curclass.out(' %s\n' % label.jasmin_syntax()) + self.curclass.out(' default: %s\n' % default.jasmin_syntax()) Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Sat Sep 15 11:35:03 2007 @@ -35,6 +35,7 @@ push_constant import pypy.translator.jvm.generator as jvmgen +from pypy.translator.jvm.log import log class Node(object): def set_db(self, db): @@ -327,6 +328,33 @@ self.ilasm.store(link.last_exc_value) self._setup_link(link) + def render_numeric_switch(self, block): + if block.exitswitch.concretetype in (ootype.SignedLongLong, ootype.UnsignedLongLong): + # TODO: it could be faster to check is the values fit in + # 32bit, and perform a cast in that case + self.render_numeric_switch_naive(block) + return + + cases, min_case, max_case, default = self._collect_switch_cases(block) + is_sparse = self._is_sparse_switch(cases, min_case, max_case) + + if is_sparse: + log.WARNING('TODO: use lookupswitch to render sparse numeric_switches') + self.render_numeric_switch_naive(block) + return + + targets = [] + for i in xrange(min_case, max_case+1): + link, lbl = cases.get(i, default) + targets.append(lbl) + + self.generator.load(block.exitswitch) + self.generator.emit_tableswitch(min_case, targets, default[1]) + + self.render_switch_case(*default) + for link, lbl in cases.itervalues(): + self.render_switch_case(link, lbl) + def render_return_block(self, block): return_var = block.inputargs[0] return_ty = self.db.lltype_to_cts(return_var.concretetype) Modified: pypy/dist/pypy/translator/oosupport/function.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/function.py (original) +++ pypy/dist/pypy/translator/oosupport/function.py Sat Sep 15 11:35:03 2007 @@ -227,6 +227,37 @@ log.WARNING("The default version of render_numeric_switch is *slow*: please override it in the backend") self.render_numeric_switch_naive(block) + def _collect_switch_cases(self, block): + cases = {} + for link in block.exits: + if link.exitcase == "default": + default = link, self.next_label('switch') + else: + if block.exitswitch.concretetype in (ootype.Char, ootype.UniChar): + value = ord(link.exitcase) + else: + value = link.exitcase + cases[value] = link, self.next_label('switch') + + values = cases.keys() + try: + min_case = min(values) + max_case = max(values) + except ValueError: + min_case = max_case = 0 + return cases, min_case, max_case, default + + def _is_sparse_switch(self, cases, min_case, max_case): + if max_case-min_case > 3*len(cases) + 10: # the switch is very sparse, better to use the naive version + return True + return False + + def render_switch_case(self, link, label): + target_label = self._get_block_name(link.target) + self.set_label(label) + self._setup_link(link) + self.generator.branch_unconditionally(target_label) + def render_numeric_switch_naive(self, block): for link in block.exits: target_label = self._get_block_name(link.target) From arigo at codespeak.net Sat Sep 15 11:40:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 11:40:08 +0200 (CEST) Subject: [pypy-svn] r46617 - pypy/dist/pypy/rpython Message-ID: <20070915094008.11FAC80C6@code0.codespeak.net> Author: arigo Date: Sat Sep 15 11:40:07 2007 New Revision: 46617 Modified: pypy/dist/pypy/rpython/rbuiltin.py Log: More cases in gen_cast(). This seems to be enough to make jit.gencode.i386.test.test_rgenop pass (didn't run all tests yet). rffi+ll2ctypes is kind of amazing... Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Sat Sep 15 11:40:07 2007 @@ -459,6 +459,20 @@ return llops.genop('cast_int_to_ptr', [v_value], resulttype=TGT) elif TGT == llmemory.Address and isinstance(ORIG, lltype.Ptr): return llops.genop('cast_ptr_to_adr', [v_value], resulttype = TGT) + elif isinstance(TGT, lltype.Primitive) and TGT in _cast_from_Signed: + if isinstance(ORIG, lltype.Ptr): + v_value = llops.genop('cast_ptr_to_int', [v_value], + resulttype=lltype.Signed) + elif ORIG == llmemory.Address: + v_value = llops.genop('cast_adr_to_int', [v_value], + resulttype=lltype.Signed) + else: + raise TypeError("don't know how to cast from %r to %r" % (ORIG, + TGT)) + op = _cast_from_Signed[TGT] + if op: + v_value = llops.genop(op, [v_value], resulttype=TGT) + return v_value raise TypeError("don't know how to cast from %r to %r" % (ORIG, TGT)) def rtype_cast_ptr_to_int(hop): From arigo at codespeak.net Sat Sep 15 12:57:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 12:57:28 +0200 (CEST) Subject: [pypy-svn] r46633 - pypy/dist/pypy/module/_socket Message-ID: <20070915105728.3D39580AE@code0.codespeak.net> Author: arigo Date: Sat Sep 15 12:57:27 2007 New Revision: 46633 Modified: pypy/dist/pypy/module/_socket/interp_func.py pypy/dist/pypy/module/_socket/interp_socket.py Log: Revert r46568 in module/_socket. Big confusion there, I completely forgot that pypy.rlib.rsocket is still based on rctypes and thus doesn't get automatic GIL release. Modified: pypy/dist/pypy/module/_socket/interp_func.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_func.py (original) +++ pypy/dist/pypy/module/_socket/interp_func.py Sat Sep 15 12:57:27 2007 @@ -10,7 +10,12 @@ Return the current host name. """ try: - res = rsocket.gethostname() + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + res = rsocket.gethostname() + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return space.wrap(res) @@ -22,7 +27,12 @@ Return the IP address (a string of the form '255.255.255.255') for a host. """ try: - addr = rsocket.gethostbyname(hostname) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + addr = rsocket.gethostbyname(hostname) + finally: + if GIL is not None: GIL.acquire(True) ip = addr.get_host() except SocketError, e: raise converted_error(space, e) @@ -43,7 +53,12 @@ for a host. The host argument is a string giving a host name or IP number. """ try: - res = rsocket.gethostbyname_ex(host) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + res = rsocket.gethostbyname_ex(host) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return common_wrapgethost(space, res) @@ -56,7 +71,12 @@ for a host. The host argument is a string giving a host name or IP number. """ try: - res = rsocket.gethostbyaddr(host) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + res = rsocket.gethostbyaddr(host) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return common_wrapgethost(space, res) @@ -74,7 +94,12 @@ else: proto = space.str_w(w_proto) try: - port = rsocket.getservbyname(name, proto) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + port = rsocket.getservbyname(name, proto) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return space.wrap(port) @@ -92,7 +117,12 @@ else: proto = space.str_w(w_proto) try: - service = rsocket.getservbyport(port, proto) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + service = rsocket.getservbyport(port, proto) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return space.wrap(service) @@ -104,7 +134,12 @@ Return the protocol number for the named protocol. (Rarely used.) """ try: - proto = rsocket.getprotobyname(name) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + proto = rsocket.getprotobyname(name) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return space.wrap(proto) @@ -115,8 +150,14 @@ Get host and port for a sockaddr.""" try: - addr = rsocket.ipaddr_from_object(space, w_sockaddr) - host, servport = rsocket.getnameinfo(addr, flags) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + addr = rsocket.ipaddr_from_object(space, w_sockaddr) + host, servport = rsocket.getnameinfo(addr, flags) + finally: + if GIL is not None: GIL.acquire(True) + except SocketError, e: raise converted_error(space, e) return space.newtuple([space.wrap(host), space.wrap(servport)]) @@ -286,8 +327,14 @@ raise OperationError(space.w_TypeError, space.wrap("Int or String expected")) try: - lst = rsocket.getaddrinfo(host, port, family, socktype, - proto, flags) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + lst = rsocket.getaddrinfo(host, port, family, socktype, + proto, flags) + finally: + if GIL is not None: GIL.acquire(True) + except SocketError, e: raise converted_error(space, e) lst1 = [space.newtuple([space.wrap(family), Modified: pypy/dist/pypy/module/_socket/interp_socket.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_socket.py (original) +++ pypy/dist/pypy/module/_socket/interp_socket.py Sat Sep 15 12:57:27 2007 @@ -30,7 +30,12 @@ info is a pair (hostaddr, port). """ try: - sock, addr = self.accept(W_RSocket) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + sock, addr = self.accept(W_RSocket) + finally: + if GIL is not None: GIL.acquire(True) return space.newtuple([space.wrap(sock), addr.as_object(space)]) except SocketError, e: @@ -45,7 +50,12 @@ sockets the address is a tuple (ifname, proto [,pkttype [,hatype]]) """ try: - self.bind(self.addr_from_object(space, w_addr)) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + self.bind(self.addr_from_object(space, w_addr)) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) bind_w.unwrap_spec = ['self', ObjSpace, W_Root] @@ -56,7 +66,12 @@ Close the socket. It cannot be used after this call. """ try: - self.close() + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + self.close() + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) close_w.unwrap_spec = ['self', ObjSpace] @@ -68,7 +83,12 @@ is a pair (host, port). """ try: - self.connect(self.addr_from_object(space, w_addr)) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + self.connect(self.addr_from_object(space, w_addr)) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) except TypeError, e: @@ -82,7 +102,12 @@ This is like connect(address), but returns an error code (the errno value) instead of raising an exception when an error occurs. """ - error = self.connect_ex(self.addr_from_object(space, w_addr)) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + error = self.connect_ex(self.addr_from_object(space, w_addr)) + finally: + if GIL is not None: GIL.acquire(True) return space.wrap(error) connect_ex_w.unwrap_spec = ['self', ObjSpace, W_Root] @@ -112,7 +137,12 @@ info is a pair (hostaddr, port). """ try: - addr = self.getpeername() + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + addr = self.getpeername() + finally: + if GIL is not None: GIL.acquire(True) return addr.as_object(space) except SocketError, e: raise converted_error(space, e) @@ -125,7 +155,12 @@ info is a pair (hostaddr, port). """ try: - addr = self.getsockname() + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + addr = self.getsockname() + finally: + if GIL is not None: GIL.acquire(True) return addr.as_object(space) except SocketError, e: raise converted_error(space, e) @@ -167,7 +202,12 @@ will allow before refusing new connections. """ try: - self.listen(backlog) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + self.listen(backlog) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) listen_w.unwrap_spec = ['self', ObjSpace, int] @@ -192,7 +232,13 @@ the remote end is closed and all data is read, return the empty string. """ try: - data = self.recv(buffersize, flags) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + data = self.recv(buffersize, flags) + finally: + if GIL is not None: GIL.acquire(True) + except SocketError, e: raise converted_error(space, e) return space.wrap(data) @@ -204,7 +250,12 @@ Like recv(buffersize, flags) but also return the sender's address info. """ try: - data, addr = self.recvfrom(buffersize, flags) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + data, addr = self.recvfrom(buffersize, flags) + finally: + if GIL is not None: GIL.acquire(True) if addr: w_addr = addr.as_object(space) else: @@ -223,7 +274,12 @@ """ data = coerce_to_str_w(space, w_data) try: - count = self.send(data, flags) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + count = self.send(data, flags) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return space.wrap(count) @@ -239,7 +295,12 @@ """ data = coerce_to_str_w(space, w_data) try: - count = self.sendall(data, flags) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + count = self.sendall(data, flags) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) sendall_w.unwrap_spec = ['self', ObjSpace, W_Root, int] @@ -260,8 +321,13 @@ flags = space.int_w(w_param2) w_addr = w_param3 try: - addr = self.addr_from_object(space, w_addr) - count = self.sendto(data, flags, addr) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + addr = self.addr_from_object(space, w_addr) + count = self.sendto(data, flags, addr) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) return space.wrap(count) @@ -325,7 +391,12 @@ (flag == SHUT_RDWR). """ try: - self.shutdown(how) + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + try: + self.shutdown(how) + finally: + if GIL is not None: GIL.acquire(True) except SocketError, e: raise converted_error(space, e) shutdown_w.unwrap_spec = ['self', ObjSpace, int] From xoraxax at codespeak.net Sat Sep 15 14:49:34 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 15 Sep 2007 14:49:34 +0200 (CEST) Subject: [pypy-svn] r46636 - in pypy/dist/pypy/translator/jvm: . src/pypy Message-ID: <20070915124934.2885880B6@code0.codespeak.net> Author: xoraxax Date: Sat Sep 15 14:49:32 2007 New Revision: 46636 Modified: pypy/dist/pypy/translator/jvm/builtin.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Log: gcj discovered a bug in genjvm - ll_contains on Strings and a char would invoke an invalid call for the String.contains method where a CharSequence would have been expected even though a Char does not implement this interface. Now there is a new helper in PyPy.java that uses indexOf. Modified: pypy/dist/pypy/translator/jvm/builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/builtin.py (original) +++ pypy/dist/pypy/translator/jvm/builtin.py Sat Sep 15 14:49:32 2007 @@ -108,9 +108,6 @@ (ootype.String.__class__, "ll_lower"): jvmgen.Method.v(jString, "toLowerCase", (), jString), - (ootype.String.__class__, "ll_contains"): - jvmgen.Method.v(jString, "contains", (jCharSequence,), jBool), - (ootype.String.__class__, "ll_replace_chr_chr"): jvmgen.Method.v(jString, "replace", (jChar, jChar), jString), Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Sat Sep 15 14:49:32 2007 @@ -565,6 +565,10 @@ return str1 + str2; } + public static boolean ll_contains(String str, char char1) { + return str.indexOf((int) char1) != -1; + } + public static int ll_find(String haystack, String needle, int start, int end) { // if it is impossible for the needle to occur: // this deals w/ a disparity in how java and python handle when needle="" From arigo at codespeak.net Sat Sep 15 15:37:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 15:37:12 +0200 (CEST) Subject: [pypy-svn] r46637 - pypy/dist/pypy/module/_file/test Message-ID: <20070915133712.DA4D480A6@code0.codespeak.net> Author: arigo Date: Sat Sep 15 15:37:12 2007 New Revision: 46637 Modified: pypy/dist/pypy/module/_file/test/test_file.py Log: These tests are really meant for "pypy-c py.test -A", where they pass. Modified: pypy/dist/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/dist/pypy/module/_file/test/test_file.py (original) +++ pypy/dist/pypy/module/_file/test/test_file.py Sat Sep 15 15:37:12 2007 @@ -1,6 +1,6 @@ import py -from pypy.conftest import gettestobjspace +from pypy.conftest import gettestobjspace, option class AppTestFile(object): def setup_class(cls): @@ -120,6 +120,8 @@ # because on top of py.py the inner calls to os.write() don't # release our object space's GIL. def setup_class(cls): + if not option.runappdirect: + py.test.skip("likely to deadlock when interpreted by py.py") cls.space = gettestobjspace(usemodules=("_file", "thread")) cls.w_temppath = cls.space.wrap( str(py.test.ensuretemp("fileimpl").join("concurrency.txt"))) @@ -150,15 +152,11 @@ f.close() def test_parallel_writes_and_reads(self): - # this test seems to be very bad: - # * when run normally, there is an early deadlock - # * when run in plain CPython (py.test -A) the print >> fwrite - # eventually deadlocks - that looks like a CPython bug - # * when run as pypy-c py.test -A, I get a Fatal RPython error, - # about an RPython-level thread.error - skip("to be looked at more closely") - - import thread, os, _file + # Warning: a test like the one below deadlocks CPython + # http://bugs.python.org/issue1164 + # It also deadlocks on py.py because the space GIL is not + # released. + import thread, sys, os, _file read_fd, write_fd = os.pipe() fread = _file.file.fdopen(read_fd, 'rb', 200) fwrite = _file.file.fdopen(write_fd, 'wb', 200) @@ -171,16 +169,20 @@ print >> fwrite, f, f = 4*f - 3*f*f print >> fwrite, "X" + fwrite.flush() + sys.stdout.write('writer ends\n') - def reader(): + def reader(j): while True: data = fread.read(1) + #sys.stdout.write('%d%r ' % (j, data)) if data == "X": break + sys.stdout.write('reader ends\n') readers_done[0] += 1 for j in range(3): - thread.start_new_thread(reader, ()) + thread.start_new_thread(reader, (j,)) thread.start_new_thread(writer, ()) import time @@ -192,9 +194,15 @@ assert readers_done[0] == 0 run = False # end the writers - while readers_done[0] != 3: - print 'readers_done == %d, still waiting...' % (readers_done[0],) + for i in range(600): time.sleep(0.4) + sys.stdout.flush() + x = readers_done[0] + if x == 3: + break + print 'readers_done == %d, still waiting...' % (x,) + else: + raise Exception("time out") print 'Passed.' From arigo at codespeak.net Sat Sep 15 15:38:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 15:38:24 +0200 (CEST) Subject: [pypy-svn] r46638 - pypy/dist/pypy/tool Message-ID: <20070915133824.B7E2B80A6@code0.codespeak.net> Author: arigo Date: Sat Sep 15 15:38:24 2007 New Revision: 46638 Modified: pypy/dist/pypy/tool/cache.py Log: Protect cache building in a multithreaded py.py. Modified: pypy/dist/pypy/tool/cache.py ============================================================================== --- pypy/dist/pypy/tool/cache.py (original) +++ pypy/dist/pypy/tool/cache.py Sat Sep 15 15:38:24 2007 @@ -25,6 +25,9 @@ # Be sure to call the parent __init__() if you override it. # +from threading import RLock +lock = RLock() # multithreading protection + class Cache(object): def __init__(self): @@ -32,20 +35,24 @@ self._building = {} def getorbuild(self, key): + lock.acquire() try: - return self.content[key] - except KeyError: - if key in self._building: - raise Exception, "%s recursive building of %r" % ( - self, key) - self._building[key] = True try: - result = self._build(key) - self.content[key] = result - finally: - del self._building[key] - self._ready(result) - return result + return self.content[key] + except KeyError: + if key in self._building: + raise Exception, "%s recursive building of %r" % ( + self, key) + self._building[key] = True + try: + result = self._build(key) + self.content[key] = result + finally: + del self._building[key] + self._ready(result) + return result + finally: + lock.release() getorbuild._annspecialcase_ = "specialize:memo" def _ready(self, result): From arigo at codespeak.net Sat Sep 15 15:39:55 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 15:39:55 +0200 (CEST) Subject: [pypy-svn] r46639 - in pypy/dist/pypy: lib module/__builtin__ module/thread module/thread/test Message-ID: <20070915133955.06C1680A6@code0.codespeak.net> Author: arigo Date: Sat Sep 15 15:39:55 2007 New Revision: 46639 Modified: pypy/dist/pypy/lib/imp.py pypy/dist/pypy/module/__builtin__/importing.py pypy/dist/pypy/module/thread/__init__.py pypy/dist/pypy/module/thread/os_thread.py pypy/dist/pypy/module/thread/test/test_import_lock.py Log: Move the import lock at interp-level. This is less of a hack, and it also has the advantage that it can be slightly finer-grained. Something easy to satisfy like 'import sys' should not block even if another thread has acquired the import lock. Modified: pypy/dist/pypy/lib/imp.py ============================================================================== --- pypy/dist/pypy/lib/imp.py (original) +++ pypy/dist/pypy/lib/imp.py Sat Sep 15 15:39:55 2007 @@ -140,73 +140,3 @@ def is_frozen(name): return False - - -# ____________________________________________________________ - -try: - # PyPy-specific interface: hint from the thread module to ask us to - # provide an import lock - from thread import _please_provide_import_lock -except ImportError: - def lock_held(): - """On platforms without threads, return False.""" - return False - def acquire_lock(): - """On platforms without threads, this function does nothing.""" - def release_lock(): - """On platforms without threads, this function does nothing.""" - -else: - del _please_provide_import_lock - import thread - - class _ImportLock: - def __init__(self): - self.lock = thread.allocate_lock() - self.in_thread = None - self.recursions = 0 - - def held(self): - """Return True if the import lock is currently held, else False.""" - return self.in_thread is not None - - def acquire(self): - """Acquires the interpreter's import lock for the current thread. - This lock should be used by import hooks to ensure thread-safety - when importing modules. - """ - myident = thread.get_ident() - if self.in_thread == myident: - self.recursions += 1 - else: - self.lock.acquire() - self.in_thread = myident - self.recursions = 1 - - def release(self): - """Release the interpreter's import lock.""" - myident = thread.get_ident() - if self.in_thread != myident: - raise RuntimeError("not holding the import lock") - self.recursions -= 1 - if self.recursions == 0: - self.in_thread = None - self.lock.release() - - _importlock = _ImportLock() - - lock_held = _importlock.held - acquire_lock = _importlock.acquire - release_lock = _importlock.release - - import __builtin__ - _original_import_hook = __builtin__.__import__ - def __lockedimport__(modulename, globals=None, locals=None, fromlist=None): - acquire_lock() - try: - return _original_import_hook(modulename, globals, locals, fromlist) - finally: - release_lock() - __builtin__.__import__ = __lockedimport__ - del __builtin__ Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Sat Sep 15 15:39:55 2007 @@ -196,6 +196,16 @@ importhook.unwrap_spec = [ObjSpace,str,W_Root,W_Root,W_Root] def absolute_import(space, modulename, baselevel, w_fromlist, tentative): + lock = getimportlock(space) + acquired = lock.try_acquire_lock() + try: + return _absolute_import(space, modulename, baselevel, + w_fromlist, tentative) + finally: + if acquired: + lock.release_lock() + +def _absolute_import(space, modulename, baselevel, w_fromlist, tentative): w = space.wrap w_mod = None @@ -286,6 +296,45 @@ # __________________________________________________________________ # +# import lock, to prevent two threads from running module-level code in +# parallel. This behavior is more or less part of the language specs, +# as an attempt to avoid failure of 'from x import y' if module x is +# still being executed in another thread. + +class ImportRLock: + def __init__(self, space): + self.space = space + self.lock = None + self.lockowner = None + + def _freeze_(self): + # remove the lock object, which will be created again as need at + # run-time. + self.lock = None + assert self.lockowner is None + return False + + def try_acquire_lock(self): + # this function runs with the GIL acquired so there is no race + # condition in the creation of the lock + if self.lock is None: + self.lock = self.space.allocate_lock() + me = self.space.getexecutioncontext() # used as thread ident + if self.lockowner is me: + return False # already acquired by the current thread + self.lock.acquire(True) + self.lockowner = me + return True + + def release_lock(self): + self.lockowner = None + self.lock.release() + +def getimportlock(space): + return space.fromcache(ImportRLock) + +# __________________________________________________________________ +# # .pyc file support """ Modified: pypy/dist/pypy/module/thread/__init__.py ============================================================================== --- pypy/dist/pypy/module/thread/__init__.py (original) +++ pypy/dist/pypy/module/thread/__init__.py Sat Sep 15 15:39:55 2007 @@ -17,7 +17,6 @@ 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.getlocktype(space)', '_local': 'os_local.getlocaltype(space)', - '_please_provide_import_lock': '(space.w_True)', # for imp.py } def __init__(self, space, *args): Modified: pypy/dist/pypy/module/thread/os_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/os_thread.py (original) +++ pypy/dist/pypy/module/thread/os_thread.py Sat Sep 15 15:39:55 2007 @@ -52,11 +52,7 @@ def setup_threads(space): - if space.threadlocals.setup_threads(space): - # the import lock is in imp.py, which registers a custom import - # hook with this lock. - from pypy.module.__builtin__.importing import importhook - importhook(space, 'imp') + space.threadlocals.setup_threads(space) def start_new_thread(space, w_callable, w_args, w_kwargs=NoneNotWrapped): Modified: pypy/dist/pypy/module/thread/test/test_import_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_import_lock.py (original) +++ pypy/dist/pypy/module/thread/test/test_import_lock.py Sat Sep 15 15:39:55 2007 @@ -13,3 +13,7 @@ thread.start_new_thread(f, ()) self.waitfor(lambda: len(done) == 5) assert len(done) == 5 + + def test_with_many_dependencies(self): + import thread + import re # -> causes nested imports From arigo at codespeak.net Sat Sep 15 16:46:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 16:46:19 +0200 (CEST) Subject: [pypy-svn] r46640 - in pypy/dist/pypy/module: __builtin__ thread/test Message-ID: <20070915144619.B4EB680BA@code0.codespeak.net> Author: arigo Date: Sat Sep 15 16:46:18 2007 New Revision: 46640 Modified: pypy/dist/pypy/module/__builtin__/importing.py pypy/dist/pypy/module/thread/test/test_import_lock.py Log: Make the import lock really more fine-grained (the previous attempt didn't achieve that). Add a test, which is really an implementation test as CPython fails it. Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Sat Sep 15 16:46:18 2007 @@ -196,16 +196,6 @@ importhook.unwrap_spec = [ObjSpace,str,W_Root,W_Root,W_Root] def absolute_import(space, modulename, baselevel, w_fromlist, tentative): - lock = getimportlock(space) - acquired = lock.try_acquire_lock() - try: - return _absolute_import(space, modulename, baselevel, - w_fromlist, tentative) - finally: - if acquired: - lock.release_lock() - -def _absolute_import(space, modulename, baselevel, w_fromlist, tentative): w = space.wrap w_mod = None @@ -258,34 +248,42 @@ w_mod = space.sys.getmodule(modulename) if w_mod is not None: return w_mod - # Examin importhooks (PEP302) before doing the import - if w_path is not None: - w_loader = find_module(space, w_modulename, w_path) - else: - w_loader = find_module(space, w_modulename, space.w_None) - if not space.is_w(w_loader, space.w_None): - w_mod = space.call_method(w_loader, "load_module", w_modulename) - #w_mod_ = check_sys_modules(space, w_modulename) - if w_mod is not None and w_parent is not None: - space.setattr(w_parent, w(partname), w_mod) - return w_mod - - - if w_path is not None: - for path in space.unpackiterable(w_path): - dir = os.path.join(space.str_w(path), partname) - if os.path.isdir(dir): - fn = os.path.join(dir, '__init__') + lock = getimportlock(space) + acquired = lock.try_acquire_lock() + try: + # Examin importhooks (PEP302) before doing the import + if w_path is not None: + w_loader = find_module(space, w_modulename, w_path) + else: + w_loader = find_module(space, w_modulename, space.w_None) + if not space.is_w(w_loader, space.w_None): + w_mod = space.call_method(w_loader, "load_module", w_modulename) + #w_mod_ = check_sys_modules(space, w_modulename) + if w_mod is not None and w_parent is not None: + space.setattr(w_parent, w(partname), w_mod) + + return w_mod + + + if w_path is not None: + for path in space.unpackiterable(w_path): + dir = os.path.join(space.str_w(path), partname) + if os.path.isdir(dir): + fn = os.path.join(dir, '__init__') + w_mod = try_import_mod(space, w_modulename, fn, + w_parent, w(partname), + pkgdir=dir) + if w_mod is not None: + return w_mod + fn = os.path.join(space.str_w(path), partname) w_mod = try_import_mod(space, w_modulename, fn, w_parent, - w(partname), pkgdir=dir) + w(partname)) if w_mod is not None: return w_mod - fn = os.path.join(space.str_w(path), partname) - w_mod = try_import_mod(space, w_modulename, fn, w_parent, - w(partname)) - if w_mod is not None: - return w_mod + finally: + if acquired: + lock.release_lock() if tentative: return None @@ -302,6 +300,8 @@ # still being executed in another thread. class ImportRLock: + _ann_seen = False + def __init__(self, space): self.space = space self.lock = None @@ -312,12 +312,14 @@ # run-time. self.lock = None assert self.lockowner is None + self._ann_seen = True return False def try_acquire_lock(self): # this function runs with the GIL acquired so there is no race # condition in the creation of the lock if self.lock is None: + assert not self._ann_seen self.lock = self.space.allocate_lock() me = self.space.getexecutioncontext() # used as thread ident if self.lockowner is me: Modified: pypy/dist/pypy/module/thread/test/test_import_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_import_lock.py (original) +++ pypy/dist/pypy/module/thread/test/test_import_lock.py Sat Sep 15 16:46:18 2007 @@ -1,8 +1,14 @@ +from pypy.tool.udir import udir from pypy.module.thread.test.support import GenericTestThread class AppTestThread(GenericTestThread): + def setup_class(cls): + GenericTestThread.setup_class.im_func(cls) + tmpdir = str(udir.ensure('test_import_lock', dir=1)) + cls.w_tmpdir = cls.space.wrap(tmpdir) + def test_import_lock(self): import thread done = [] @@ -17,3 +23,24 @@ def test_with_many_dependencies(self): import thread import re # -> causes nested imports + + def test_no_lock_for_reimporting(self): + # CPython deadlocks in this situation; in our case the property + # of not requiring the import lock for already-imported modules + # is useful for translation, to avoid needing a prebuilt import + # lock object. + import os, sys + f = open(os.path.join(self.tmpdir, 'foobaz1.py'), 'w') + print >> f, """if 1: + import thread + lock = thread.allocate_lock() + lock.acquire() + def f(): + import sys + lock.release() + thread.start_new_thread(f, ()) + lock.acquire() + """ + f.close() + sys.path.insert(0, self.tmpdir) + import foobaz1 From cfbolz at codespeak.net Sat Sep 15 23:11:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 15 Sep 2007 23:11:40 +0200 (CEST) Subject: [pypy-svn] r46662 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20070915211140.9A5F180AE@code0.codespeak.net> Author: cfbolz Date: Sat Sep 15 23:11:39 2007 New Revision: 46662 Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py Log: Fix failing tests - sorry for that. Some exception objects need to be preallocated because the inliner knows now that malloc can raise MemoryError. One ootype test couldn't be fixed easily because graphanalyze doesn't deal with oosend at all. This should be fixed, it will help other things as well (exception transformer, e.g). Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_inline.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_inline.py Sat Sep 15 23:11:39 2007 @@ -214,11 +214,13 @@ assert len(s) > 3 def test_inline_exceptions(self): + customError1 = CustomError1() + customError2 = CustomError2() def f(x): if x == 0: - raise CustomError1 + raise customError1 if x == 1: - raise CustomError2 + raise customError2 def g(x): try: f(x) @@ -484,8 +486,9 @@ assert result == 55 def test_inline_raiseonly(self): + c = CustomError1() def f2(x): - raise CustomError1 + raise c def f(x): try: return f2(x) @@ -541,6 +544,7 @@ py.test.raises(CannotInline, self.check_inline, x3, x4, []) def test_list_iteration(self): + self._skip_oo("graphanalyze properly!") def f(): tot = 0 for item in [1,2,3]: @@ -619,7 +623,6 @@ assert res == 42 def test_float(self): - py.test.skip('fixme!') ex = ['', ' '] def fn(i): s = ex[i] From cfbolz at codespeak.net Sat Sep 15 23:32:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 15 Sep 2007 23:32:19 +0200 (CEST) Subject: [pypy-svn] r46663 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20070915213219.5BDB280B9@code0.codespeak.net> Author: cfbolz Date: Sat Sep 15 23:32:19 2007 New Revision: 46663 Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py Log: add a comment Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_canraise.py Sat Sep 15 23:32:19 2007 @@ -91,6 +91,8 @@ assert result def test_instantiate(): + # instantiate is interesting, because it leads to one of the few cases of + # an indirect call without a list of graphs from pypy.rlib.objectmodel import instantiate class A: pass From arigo at codespeak.net Sat Sep 15 23:39:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 15 Sep 2007 23:39:32 +0200 (CEST) Subject: [pypy-svn] r46664 - pypy/dist/pypy/module/__builtin__ Message-ID: <20070915213932.379E880B9@code0.codespeak.net> Author: arigo Date: Sat Sep 15 23:39:17 2007 New Revision: 46664 Modified: pypy/dist/pypy/module/__builtin__/importing.py Log: Oups! The "assert False" is hard-coded into the C code. Bad move. Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Sat Sep 15 23:39:17 2007 @@ -11,6 +11,7 @@ from pypy.interpreter.eval import Code from pypy.rlib import streamio from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import we_are_translated NOFILE = 0 PYFILE = 1 @@ -319,7 +320,12 @@ # this function runs with the GIL acquired so there is no race # condition in the creation of the lock if self.lock is None: - assert not self._ann_seen + # sanity-check: imports should not occur after the + # ImportRLock has been frozen (and self.lock thrown away) + # during annotation. They can occur either before, or in + # the translated code. + if not we_are_translated(): + assert not self._ann_seen self.lock = self.space.allocate_lock() me = self.space.getexecutioncontext() # used as thread ident if self.lockowner is me: From cfbolz at codespeak.net Sat Sep 15 23:40:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 15 Sep 2007 23:40:10 +0200 (CEST) Subject: [pypy-svn] r46665 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20070915214010.72DB780B9@code0.codespeak.net> Author: cfbolz Date: Sat Sep 15 23:40:10 2007 New Revision: 46665 Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py Log: refactor canraise tests to be run on top of ootype too Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_canraise.py Sat Sep 15 23:40:10 2007 @@ -1,112 +1,124 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.canraise import RaiseAnalyzer from pypy.translator.backendopt.all import backend_optimizations +from pypy.rpython.test.tool import LLRtypeMixin, OORtypeMixin from pypy.conftest import option -def translate(func, sig): - t = TranslationContext() - t.buildannotator().build_types(func, sig) - t.buildrtyper().specialize() - if option.view: - t.view() - return t, RaiseAnalyzer(t) - -def test_can_raise_simple(): - def g(x): - return True - - def f(x): - return g(x - 1) - t, ra = translate(f, [int]) - fgraph = graphof(t, f) - result = ra.can_raise(fgraph.startblock.operations[0]) - assert not result - -def test_can_raise_recursive(): - from pypy.translator.transform import insert_ll_stackcheck - def g(x): - return f(x) +class BaseTestCanRaise(object): + type_system = None - def f(x): - if x: + def translate(self, func, sig): + t = TranslationContext() + t.buildannotator().build_types(func, sig) + t.buildrtyper(type_system=self.type_system).specialize() + if option.view: + t.view() + return t, RaiseAnalyzer(t) + + + def test_can_raise_simple(self): + def g(x): + return True + + def f(x): return g(x - 1) - return 1 - t, ra = translate(f, [int]) - insert_ll_stackcheck(t) - ggraph = graphof(t, g) - result = ra.can_raise(ggraph.startblock.operations[-1]) - assert result # due to stack check every recursive function can raise - -def test_can_raise_exception(): - def g(): - raise ValueError - def f(): - return g() - t, ra = translate(f, []) - fgraph = graphof(t, f) - result = ra.can_raise(fgraph.startblock.operations[0]) - assert result - -def test_indirect_call(): - def g1(): - raise ValueError - def g2(): - return 2 - def f(x): - if x: - g = g1 - else: - g = g2 - return g() - def h(x): - return f(x) - t, ra = translate(h, [int]) - hgraph = graphof(t, h) - result = ra.can_raise(hgraph.startblock.operations[0]) - assert result - -def test_llexternal(): - from pypy.rpython.lltypesystem.rffi import llexternal - from pypy.rpython.lltypesystem import lltype - z = llexternal('z', [lltype.Signed], lltype.Signed) - def f(x): - return z(x) - t, ra = translate(f, [int]) - fgraph = graphof(t, f) - backend_optimizations(t) - assert fgraph.startblock.operations[0].opname == 'direct_call' - - result = ra.can_raise(fgraph.startblock.operations[0]) - assert not result - - z = llexternal('z', [lltype.Signed], lltype.Signed, canraise=True) - def g(x): - return z(x) - t, ra = translate(g, [int]) - ggraph = graphof(t, g) - - assert ggraph.startblock.operations[0].opname == 'direct_call' - - result = ra.can_raise(ggraph.startblock.operations[0]) - assert result - -def test_instantiate(): - # instantiate is interesting, because it leads to one of the few cases of - # an indirect call without a list of graphs - from pypy.rlib.objectmodel import instantiate - class A: - pass - class B(A): - pass - def g(x): - if x: - C = A - else: - C = B - a = instantiate(C) - def f(x): - return g(x) - t, ra = translate(f, [int]) - fgraph = graphof(t, f) - result = ra.can_raise(fgraph.startblock.operations[0]) - assert result + t, ra = self.translate(f, [int]) + fgraph = graphof(t, f) + result = ra.can_raise(fgraph.startblock.operations[0]) + assert not result + + def test_can_raise_recursive(self): + from pypy.translator.transform import insert_ll_stackcheck + def g(x): + return f(x) + + def f(x): + if x: + return g(x - 1) + return 1 + t, ra = self.translate(f, [int]) + insert_ll_stackcheck(t) + ggraph = graphof(t, g) + result = ra.can_raise(ggraph.startblock.operations[-1]) + assert result # due to stack check every recursive function can raise + + def test_can_raise_exception(self): + def g(): + raise ValueError + def f(): + return g() + t, ra = self.translate(f, []) + fgraph = graphof(t, f) + result = ra.can_raise(fgraph.startblock.operations[0]) + assert result + + def test_indirect_call(self): + def g1(): + raise ValueError + def g2(): + return 2 + def f(x): + if x: + g = g1 + else: + g = g2 + return g() + def h(x): + return f(x) + t, ra = self.translate(h, [int]) + hgraph = graphof(t, h) + result = ra.can_raise(hgraph.startblock.operations[0]) + assert result + + def test_instantiate(self): + # instantiate is interesting, because it leads to one of the few cases of + # an indirect call without a list of graphs + from pypy.rlib.objectmodel import instantiate + class A: + pass + class B(A): + pass + def g(x): + if x: + C = A + else: + C = B + a = instantiate(C) + def f(x): + return g(x) + t, ra = self.translate(f, [int]) + fgraph = graphof(t, f) + result = ra.can_raise(fgraph.startblock.operations[0]) + assert result + +class TestLLType(LLRtypeMixin, BaseTestCanRaise): + def test_llexternal(self): + from pypy.rpython.lltypesystem.rffi import llexternal + from pypy.rpython.lltypesystem import lltype + z = llexternal('z', [lltype.Signed], lltype.Signed) + def f(x): + return z(x) + t, ra = self.translate(f, [int]) + fgraph = graphof(t, f) + backend_optimizations(t) + assert fgraph.startblock.operations[0].opname == 'direct_call' + + result = ra.can_raise(fgraph.startblock.operations[0]) + assert not result + + z = llexternal('z', [lltype.Signed], lltype.Signed, canraise=True) + def g(x): + return z(x) + t, ra = self.translate(g, [int]) + ggraph = graphof(t, g) + + assert ggraph.startblock.operations[0].opname == 'direct_call' + + result = ra.can_raise(ggraph.startblock.operations[0]) + assert result + + +class TestOOType(OORtypeMixin, BaseTestCanRaise): + pass + + From cfbolz at codespeak.net Sat Sep 15 23:55:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 15 Sep 2007 23:55:42 +0200 (CEST) Subject: [pypy-svn] r46667 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20070915215542.C414A80B9@code0.codespeak.net> Author: cfbolz Date: Sat Sep 15 23:55:42 2007 New Revision: 46667 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py Log: correct llop table Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Sat Sep 15 23:55:42 2007 @@ -402,9 +402,8 @@ # __________ weakrefs __________ - # weakref_create can raise MemoryError in GCs like Boehm, but not - # in the framework GC, so it should never cause a stack unwind. - 'weakref_create': LLOp(canraise=(MemoryError,), sideeffects=False), + 'weakref_create': LLOp(canraise=(MemoryError,), sideeffects=False, + canunwindgc=True), 'weakref_deref': LLOp(sideeffects=False), 'cast_ptr_to_weakrefptr': LLOp(sideeffects=False), # no-op type hiding 'cast_weakrefptr_to_ptr': LLOp(sideeffects=False), # no-op type revealing From exarkun at codespeak.net Sun Sep 16 02:26:44 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sun, 16 Sep 2007 02:26:44 +0200 (CEST) Subject: [pypy-svn] r46668 - pypy/branch/zlib-rffi-impl Message-ID: <20070916002644.DBA1680D3@code0.codespeak.net> Author: exarkun Date: Sun Sep 16 02:26:43 2007 New Revision: 46668 Added: pypy/branch/zlib-rffi-impl/ - copied from r46667, pypy/dist/ Log: Branching to implement zlib with rffi From exarkun at codespeak.net Sun Sep 16 02:29:34 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sun, 16 Sep 2007 02:29:34 +0200 (CEST) Subject: [pypy-svn] r46669 - in pypy/branch/zlib-rffi-impl/pypy/module/zlib: . test Message-ID: <20070916002934.BE22980D6@code0.codespeak.net> Author: exarkun Date: Sun Sep 16 02:29:34 2007 New Revision: 46669 Added: pypy/branch/zlib-rffi-impl/pypy/module/zlib/ pypy/branch/zlib-rffi-impl/pypy/module/zlib/__init__.py pypy/branch/zlib-rffi-impl/pypy/module/zlib/app_zlib.py pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py pypy/branch/zlib-rffi-impl/pypy/module/zlib/test/ pypy/branch/zlib-rffi-impl/pypy/module/zlib/test/test_zlib.py Log: first attempt Added: pypy/branch/zlib-rffi-impl/pypy/module/zlib/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/zlib-rffi-impl/pypy/module/zlib/__init__.py Sun Sep 16 02:29:34 2007 @@ -0,0 +1,38 @@ + +""" +Mixed-module definition for the zlib module. +""" + +import zlib + +from pypy.interpreter.mixedmodule import MixedModule + +def constant(value): + return 'space.wrap(%s)' % (value,) + + +class Module(MixedModule): + interpleveldefs = { + 'crc32': 'interp_zlib.crc32', + 'adler32': 'interp_zlib.adler32', + 'Compress': 'interp_zlib.Compress', + 'Decompress': 'interp_zlib.Decompress', + } + +# # Constants exposed by zlib.h +# interpleveldefs.update(( +# (name, constant(getattr(zlib, name))) +# for name +# in ['DEFLATED', 'DEF_MEM_LEVEL', 'MAX_WBITS', +# 'Z_BEST_COMPRESSION', 'Z_BEST_SPEED', +# 'Z_DEFAULT_COMPRESSION', 'Z_DEFAULT_STRATEGY', +# 'Z_FILTERED', 'Z_FINISH', 'Z_FULL_FLUSH', +# 'Z_HUFFMAN_ONLY', 'Z_NO_FLUSH', 'Z_SYNC_FLUSH'])) + + appleveldefs = { + 'error': 'app_zlib.error', + 'compress': 'app_zlib.compress', + 'decompress': 'app_zlib.decompress', + 'compressobj': 'app_zlib.compressobj', + 'decompressobj': 'app_zlib.decompressobj', + } Added: pypy/branch/zlib-rffi-impl/pypy/module/zlib/app_zlib.py ============================================================================== --- (empty file) +++ pypy/branch/zlib-rffi-impl/pypy/module/zlib/app_zlib.py Sun Sep 16 02:29:34 2007 @@ -0,0 +1,54 @@ + +""" +Application-level definitions for the zlib module. + +NOT_RPYTHON +""" + +class error(Exception): + """ + Raised by zlib operations. + """ + + +def compressobj(level=None): + """ + compressobj([level]) -- Return a compressor object. + + Optional arg level is the compression level, in 1-9. + """ + import zlib + return zlib.Compress(level) + + + +def decompressobj(wbits=None): + """ + decompressobj([wbits]) -- Return a decompressor object. + + Optional arg wbits is the window buffer size. + """ + import zlib + return zlib.Decompress(wbits) + + +def compress(string, level=None): + """ + compress(string[, level]) -- Returned compressed string. + + Optional arg level is the compression level, in 1-9. + """ + compressor = compressobj(level) + return compressor.compress(string) + compressor.flush() + + +def decompress(string, wbits=None, bufsize=None): + """ + decompress(string[, wbits[, bufsize]]) -- Return decompressed string. + + Optional arg wbits is the window buffer size. Optional arg bufsize is + the initial output buffer size. + """ + # XXX bufsize is ignored because it's basically useless. + decompressor = decompressobj(wbits) + return decompressor.decompress(string) + decompressor.flush() Added: pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py ============================================================================== --- (empty file) +++ pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py Sun Sep 16 02:29:34 2007 @@ -0,0 +1,316 @@ +# import zlib + +from pypy.interpreter.gateway import ObjSpace, interp2app +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.error import OperationError + +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.tool import rffi_platform + +includes = ['zlib.h'] +libraries = ['z'] + +class SimpleCConfig: + """ + Definitions for basic types defined by zlib. + """ + _includes_ = includes + + # XXX If Z_PREFIX was defined for the libz build, then these types are + # named z_uInt, z_uLong, and z_Bytef instead. + uInt = rffi_platform.SimpleType('uInt', rffi.UINT) + uLong = rffi_platform.SimpleType('uLong', rffi.ULONG) + Bytef = rffi_platform.SimpleType('Bytef', rffi.UCHAR) + voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP) + + Z_OK = rffi_platform.ConstantInteger('Z_OK') + Z_STREAM_ERROR = rffi_platform.ConstantInteger('Z_STREAM_ERROR') + + ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION') + Z_DEFAULT_COMPRESSION = rffi_platform.ConstantInteger( + 'Z_DEFAULT_COMPRESSION') + Z_NO_FLUSH = rffi_platform.ConstantInteger( + 'Z_NO_FLUSH') + +config = rffi_platform.configure(SimpleCConfig) +voidpf = config['voidpf'] +uInt = config['uInt'] +uLong = config['uLong'] +Bytef = config['Bytef'] +Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True})) + +Z_OK = config['Z_OK'] +Z_STREAM_ERROR = config['Z_STREAM_ERROR'] + +ZLIB_VERSION = config['ZLIB_VERSION'] +Z_DEFAULT_COMPRESSION = config['Z_DEFAULT_COMPRESSION'] +Z_NO_FLUSH = config['Z_DEFAULT_COMPRESSION'] + +class ComplexCConfig: + """ + Definitions of structure types defined by zlib and based on SimpleCConfig + definitions. + """ + _includes_ = includes + + z_stream = rffi_platform.Struct( + 'z_stream', + [('next_in', Bytefp), + ('avail_in', uInt), + ('total_in', uLong), + + ('next_out', Bytefp), + ('avail_out', uInt), + ('total_out', uLong), + + ('msg', rffi.CCHARP), + + ('zalloc', lltype.Ptr( + lltype.FuncType([voidpf, uInt, uInt], voidpf))), + ('zfree', lltype.Ptr( + lltype.FuncType([voidpf, voidpf], lltype.Void))), + + ('opaque', voidpf), + + ('data_type', rffi.INT), + ('adler', uLong), + ('reserved', uLong) + ]) + +config = rffi_platform.configure(ComplexCConfig) +z_stream = config['z_stream'] +z_stream_p = lltype.Ptr(z_stream) + +def zlib_external(*a, **kw): + kw['includes'] = includes + kw['libraries'] = libraries + return rffi.llexternal(*a, **kw) + +_crc32 = zlib_external('crc32', [uLong, Bytefp, uInt], uLong) +_adler32 = zlib_external('adler32', [uLong, Bytefp, uInt], uLong) +_deflateInit = zlib_external( + 'deflateInit_', [z_stream_p, rffi.INT, rffi.CCHARP, rffi.INT], + rffi.INT) +_deflate = zlib_external('deflate', [z_stream_p, rffi.INT], rffi.INT) + +def crc32(space, string, start=0): + """ + crc32(string[, start]) -- Compute a CRC-32 checksum of string. + + An optional starting value can be specified. The returned checksum is + an integer. + """ + bytes = rffi.str2charp(string) + checksum = _crc32(start, rffi.cast(Bytefp, bytes), len(string)) + rffi.free_charp(bytes) + + # This is, perhaps, a little stupid. zlib returns the checksum unsigned. + # CPython exposes it as a signed value, though. -exarkun + checksum = rffi.cast(rffi.INT, checksum) + + return space.wrap(checksum) +crc32.unwrap_spec = [ObjSpace, str, int] + + +def adler32(space, string, start=1): + """ + adler32(string[, start]) -- Compute an Adler-32 checksum of string. + + An optional starting value can be specified. The returned checksum is + an integer. + """ + bytes = rffi.str2charp(string) + checksum = _adler32(start, rffi.cast(Bytefp, bytes), len(string)) + rffi.free_charp(bytes) + + # This is, perhaps, a little stupid. zlib returns the checksum unsigned. + # CPython exposes it as a signed value, though. -exarkun + checksum = rffi.cast(rffi.INT, checksum) + + return space.wrap(checksum) +adler32.unwrap_spec = [ObjSpace, str, int] + + +class _StreamBase(Wrappable): + """ + Base for classes which want to have a z_stream allocated when they are + initialized and de-allocated when they are freed. + """ + def __init__(self, space): + self.space = space + self.stream = lltype.malloc(z_stream, flavor='raw') + self.stream.c_zalloc = lltype.nullptr(z_stream.c_zalloc.TO) + self.stream.c_zfree = lltype.nullptr(z_stream.c_zfree.TO) + self.stream.c_avail_in = rffi.cast(lltype.Unsigned, 0) + + + def __del__(self): + lltype.free(self.stream, flavor='raw') + + + +def error_from_zlib(space, status): + if status == Z_STREAM_ERROR: + return OperationError( + space.w_ValueError, + space.wrap("Invalid initialization option")) + assert False, "unhandled status %s" % (status,) + + +class Compress(_StreamBase): + """ + Wrapper around zlib's z_stream structure which provides convenient + compression functionality. + """ + def __init__(self, space, w_level): + # XXX CPython actually exposes 4 more undocumented parameters beyond + # level. + if space.is_w(w_level, space.w_None): + level = Z_DEFAULT_COMPRESSION + else: + level = space.int_w(w_level) + + _StreamBase.__init__(self, space) + + # XXX I want to call deflateInit, not deflateInit_ + size = z_stream._hints['size'] + version = rffi.str2charp(ZLIB_VERSION) + result = _deflateInit(self.stream, level, version, size) + rffi.free_charp(version) + + if result != Z_OK: + raise error_from_zlib(self.space, result) + + + def compress(self, data, length=16384): + """ + compress(data) -- Return a string containing data compressed. + + After calling this function, some of the input data may still be stored + in internal buffers for later processing. + + Call the flush() method to clear these buffers. + """ + self.stream.c_avail_in = rffi.cast(lltype.Unsigned, len(data)) + self.stream.c_next_in = lltype.malloc( + Bytefp.TO, len(data), flavor='raw') + for i in xrange(len(data)): + self.stream.c_next_in[i] = rffi.cast(Bytef, data[i]) + try: + self.stream.c_avail_out = rffi.cast(lltype.Unsigned, length) + self.stream.c_next_out = lltype.malloc( + Bytefp.TO, length, flavor='raw') + try: + result = _deflate(self.stream, Z_NO_FLUSH) + if result != Z_OK: + raise error_from_zlib(self.space, result) + else: + return rffi.charp2str(self.stream.c_next_out) + finally: + lltype.free(self.stream.c_next_out, flavor='raw') + self.stream.c_avail_out = rffi.cast(lltype.Unsigned, 0) + finally: + lltype.free(self.stream.c_next_in, flavor='raw') + self.stream.c_avail_in = rffi.cast(lltype.Unsigned, 0) + compress.unwrap_spec = ['self', str, int] + + + def flush(self, mode=0): # XXX =Z_FINISH + """ + flush( [mode] ) -- Return a string containing any remaining compressed + data. + + mode can be one of the constants Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH; + the default value used when mode is not specified is Z_FINISH. + + If mode == Z_FINISH, the compressor object can no longer be used after + calling the flush() method. Otherwise, more data can still be + compressed. + """ + return self.space.wrap('') + flush.unwrap_spec = ['self', int] + + +def Compress___new__(space, w_subtype, w_anything=None): + """ + Create a new z_stream and call its initializer. + """ + stream = space.allocate_instance(Compress, w_subtype) + stream = space.interp_w(Compress, stream) + Compress.__init__(stream, space, w_anything) + return space.wrap(stream) + + +Compress.typedef = TypeDef( + 'Compress', + __new__ = interp2app(Compress___new__), + compress = interp2app(Compress.compress), + flush = interp2app(Compress.flush)) + + + +class Decompress(_StreamBase): + """ + Wrapper around zlib's z_stream structure which provides convenient + decompression functionality. + """ + def __init__(self, space, wbits): + """ + Initialize a new decompression object. + + wbits is an integer between 8 and MAX_WBITS or -8 and -MAX_WBITS + (inclusive) giving the number of "window bits" to use for compression + and decompression. See the documentation for deflateInit2 and + inflateInit2. + """ + _StreamBase.__init__(self, space) + self.wbits = wbits + + + def decompress(self, data, max_length=0): # XXX =None + """ + decompress(data[, max_length]) -- Return a string containing the + decompressed version of the data. + + After calling this function, some of the input data may still be stored + in internal buffers for later processing. + + Call the flush() method to clear these buffers. + + If the max_length parameter is specified then the return value will be + no longer than max_length. Unconsumed input data will be stored in the + unconsumed_tail attribute. + """ + return self.space.wrap('') + decompress.unwrap_spec = ['self', str, int] + + + def flush(self, length=0): + """ + flush( [length] ) -- Return a string containing any remaining + decompressed data. length, if given, is the initial size of the output + buffer. + + The decompressor object can no longer be used after this call. + """ + return self.space.wrap('') + flush.unwrap_spec = ['self', int] + + +def Decompress___new__(space, w_subtype, w_anything=None): + """ + Create a new Decompress and call its initializer. + """ + stream = space.allocate_instance(Decompress, w_subtype) + stream = space.interp_w(Decompress, stream) + Decompress.__init__(stream, space, w_anything) + return space.wrap(stream) + + + +Decompress.typedef = TypeDef( + 'Decompress', + __new__ = interp2app(Decompress___new__), + decompress = interp2app(Decompress.decompress), + flush = interp2app(Decompress.flush)) Added: pypy/branch/zlib-rffi-impl/pypy/module/zlib/test/test_zlib.py ============================================================================== --- (empty file) +++ pypy/branch/zlib-rffi-impl/pypy/module/zlib/test/test_zlib.py Sun Sep 16 02:29:34 2007 @@ -0,0 +1,115 @@ + +""" +Tests for the zlib module. +""" + +import zlib + +from pypy.conftest import gettestobjspace + +class AppTestZlib(object): + def setup_class(cls): + """ + Create a space with the zlib module and import it for use by the tests. + Also create some compressed data with the bootstrap zlib module so that + compression and decompression tests have a little real data to assert + against. + """ + cls.space = gettestobjspace(usemodules=['zlib']) + cls.w_zlib = cls.space.call_function( + cls.space.builtin.get('__import__'), + cls.space.wrap('zlib')) + expanded = 'some bytes which will be compressed' + cls.w_expanded = cls.space.wrap(expanded) + cls.w_compressed = cls.space.wrap(zlib.compress(expanded)) + + + def test_error(self): + """ + zlib.error should be an exception class. + """ + assert issubclass(self.zlib.error, Exception) + + + def test_crc32(self): + """ + When called with a string, zlib.crc32 should compute its CRC32 and + return it as a signed 32 bit integer. + """ + assert self.zlib.crc32('') == 0 + assert self.zlib.crc32('\0') == -771559539 + assert self.zlib.crc32('hello, world.') == -936931198 + + + def test_crc32_start_value(self): + """ + When called with a string and an integer, zlib.crc32 should compute the + CRC32 of the string using the integer as the starting value. + """ + assert self.zlib.crc32('', 42) == 42 + assert self.zlib.crc32('\0', 42) == 163128923 + assert self.zlib.crc32('hello, world.', 42) == 1090960721 + hello = 'hello, ' + hellocrc = self.zlib.crc32(hello) + world = 'world.' + helloworldcrc = self.zlib.crc32(world, hellocrc) + assert helloworldcrc == self.zlib.crc32(hello + world) + + + def test_adler32(self): + """ + When called with a string, zlib.crc32 should compute its adler 32 + checksum and return it as a signed 32 bit integer. + """ + assert self.zlib.adler32('') == 1 + assert self.zlib.adler32('\0') == 65537 + assert self.zlib.adler32('hello, world.') == 571147447 + assert self.zlib.adler32('x' * 23) == -2122904887 + + + def test_adler32_start_value(self): + """ + When called with a string and an integer, zlib.adler32 should compute + the adler 32 checksum of the string using the integer as the starting + value. + """ + assert self.zlib.adler32('', 42) == 42 + assert self.zlib.adler32('\0', 42) == 2752554 + assert self.zlib.adler32('hello, world.', 42) == 606078176 + assert self.zlib.adler32('x' * 23, 42) == -2061104398 + hello = 'hello, ' + hellosum = self.zlib.adler32(hello) + world = 'world.' + helloworldsum = self.zlib.adler32(world, hellosum) + assert helloworldsum == self.zlib.adler32(hello + world) + + + def test_invalidLevel(self): + """ + zlib.compressobj should raise ValueError when an out of bounds level is + passed to it. + """ + raises(ValueError, self.zlib.compressobj, -2) + raises(ValueError, self.zlib.compressobj, 10) + + + def test_compression(self): + """ + zlib.compressobj should return an object which can be used to compress + bytes. + """ + compressor = self.zlib.compressobj() + bytes = compressor.compress(self.expanded) + bytes += compressor.flush() + assert bytes == self.compressed + + + def test_decompression(self): + """ + zlib.decompressobj should return an object which can be used to + decompress bytes. + """ + decompressor = self.zlib.decompressobj() + bytes = decompressor.decompress(self.compressed) + bytes += decompressor.flush() + assert bytes == self.expanded From arigo at codespeak.net Sun Sep 16 11:52:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 11:52:58 +0200 (CEST) Subject: [pypy-svn] r46673 - pypy/dist/pypy/jit/codegen/i386/test Message-ID: <20070916095258.2113D80C0@code0.codespeak.net> Author: arigo Date: Sun Sep 16 11:52:57 2007 New Revision: 46673 Modified: pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py Log: An extra test for 'free', which is otherwise only called by a __del__ (exceptions there are ignored and don't show up as test failures). Modified: pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py Sun Sep 16 11:52:57 2007 @@ -1,7 +1,16 @@ from pypy.jit.codegen.i386.ri386 import * -from pypy.jit.codegen.i386.codebuf import MachineCodeBlock +from pypy.jit.codegen.i386.codebuf import MachineCodeBlock, memhandler +def test_alloc_free(): + map_size = 65536 + data = memhandler.alloc(map_size) + for i in range(0, map_size, 171): + data[i] = chr(i & 0xff) + for i in range(0, map_size, 171): + assert data[i] == chr(i & 0xff) + memhandler.free(data, map_size) + def test_machinecodeblock(): mc = MachineCodeBlock(4096) mc.MOV(eax, mem(esp, 4)) From arigo at codespeak.net Sun Sep 16 12:54:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 12:54:24 +0200 (CEST) Subject: [pypy-svn] r46674 - in pypy/dist/pypy: lib module/__builtin__ module/thread module/thread/test Message-ID: <20070916105424.6554180AB@code0.codespeak.net> Author: arigo Date: Sun Sep 16 12:54:23 2007 New Revision: 46674 Added: pypy/dist/pypy/module/thread/importlock.py (contents, props changed) Modified: pypy/dist/pypy/lib/imp.py pypy/dist/pypy/module/__builtin__/importing.py pypy/dist/pypy/module/thread/__init__.py pypy/dist/pypy/module/thread/test/test_import_lock.py Log: * reimplement imp.{acquire,release}_lock(), as needed by lib/runpy.py. * the "fine-grained locking" idea was too naive. Reverted to regular locking around almost the whole __import__ hook. * cope with this with a different workaround against prebuilt locks. Modified: pypy/dist/pypy/lib/imp.py ============================================================================== --- pypy/dist/pypy/lib/imp.py (original) +++ pypy/dist/pypy/lib/imp.py Sun Sep 16 12:54:23 2007 @@ -140,3 +140,19 @@ def is_frozen(name): return False + +# ____________________________________________________________ + +try: + # PyPy-specific interface + from thread import _importlock_held as lock_held + from thread import _importlock_acquire as acquire_lock + from thread import _importlock_release as release_lock +except ImportError: + def lock_held(): + """On platforms without threads, return False.""" + return False + def acquire_lock(): + """On platforms without threads, this function does nothing.""" + def release_lock(): + """On platforms without threads, this function does nothing.""" Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Sun Sep 16 12:54:23 2007 @@ -197,6 +197,15 @@ importhook.unwrap_spec = [ObjSpace,str,W_Root,W_Root,W_Root] def absolute_import(space, modulename, baselevel, w_fromlist, tentative): + lock = getimportlock(space) + lock.acquire_lock() + try: + return _absolute_import(space, modulename, baselevel, + w_fromlist, tentative) + finally: + lock.release_lock() + +def _absolute_import(space, modulename, baselevel, w_fromlist, tentative): w = space.wrap w_mod = None @@ -246,45 +255,35 @@ if not space.is_w(w_mod, space.w_None): return w_mod else: - w_mod = space.sys.getmodule(modulename) - if w_mod is not None: + # Examin importhooks (PEP302) before doing the import + if w_path is not None: + w_loader = find_module(space, w_modulename, w_path) + else: + w_loader = find_module(space, w_modulename, space.w_None) + if not space.is_w(w_loader, space.w_None): + w_mod = space.call_method(w_loader, "load_module", w_modulename) + #w_mod_ = check_sys_modules(space, w_modulename) + if w_mod is not None and w_parent is not None: + space.setattr(w_parent, w(partname), w_mod) + return w_mod - lock = getimportlock(space) - acquired = lock.try_acquire_lock() - try: - # Examin importhooks (PEP302) before doing the import - if w_path is not None: - w_loader = find_module(space, w_modulename, w_path) - else: - w_loader = find_module(space, w_modulename, space.w_None) - if not space.is_w(w_loader, space.w_None): - w_mod = space.call_method(w_loader, "load_module", w_modulename) - #w_mod_ = check_sys_modules(space, w_modulename) - if w_mod is not None and w_parent is not None: - space.setattr(w_parent, w(partname), w_mod) - - return w_mod - - - if w_path is not None: - for path in space.unpackiterable(w_path): - dir = os.path.join(space.str_w(path), partname) - if os.path.isdir(dir): - fn = os.path.join(dir, '__init__') - w_mod = try_import_mod(space, w_modulename, fn, - w_parent, w(partname), - pkgdir=dir) - if w_mod is not None: - return w_mod - fn = os.path.join(space.str_w(path), partname) - w_mod = try_import_mod(space, w_modulename, fn, w_parent, - w(partname)) + + if w_path is not None: + for path in space.unpackiterable(w_path): + dir = os.path.join(space.str_w(path), partname) + if os.path.isdir(dir): + fn = os.path.join(dir, '__init__') + w_mod = try_import_mod(space, w_modulename, fn, + w_parent, w(partname), + pkgdir=dir) if w_mod is not None: return w_mod - finally: - if acquired: - lock.release_lock() + fn = os.path.join(space.str_w(path), partname) + w_mod = try_import_mod(space, w_modulename, fn, w_parent, + w(partname)) + if w_mod is not None: + return w_mod if tentative: return None @@ -300,43 +299,63 @@ # as an attempt to avoid failure of 'from x import y' if module x is # still being executed in another thread. +# This logic is tested in pypy.module.thread.test.test_import_lock. + class ImportRLock: - _ann_seen = False def __init__(self, space): self.space = space self.lock = None self.lockowner = None + self.lockcounter = 0 - def _freeze_(self): - # remove the lock object, which will be created again as need at - # run-time. - self.lock = None - assert self.lockowner is None - self._ann_seen = True - return False + def lock_held(self): + me = self.space.getexecutioncontext() # used as thread ident + return self.lockowner is me - def try_acquire_lock(self): + def _can_have_lock(self): + # hack: we can't have self.lock != None during translation, + # because prebuilt lock objects are not allowed. In this + # special situation we just don't lock at all (translation is + # not multithreaded anyway). + if we_are_translated(): + return True # we need a lock at run-time + elif self.space.config.translating: + assert self.lock is None + return False + else: + return True # in py.py + + def acquire_lock(self): # this function runs with the GIL acquired so there is no race # condition in the creation of the lock if self.lock is None: - # sanity-check: imports should not occur after the - # ImportRLock has been frozen (and self.lock thrown away) - # during annotation. They can occur either before, or in - # the translated code. - if not we_are_translated(): - assert not self._ann_seen + if not self._can_have_lock(): + return self.lock = self.space.allocate_lock() me = self.space.getexecutioncontext() # used as thread ident if self.lockowner is me: - return False # already acquired by the current thread - self.lock.acquire(True) - self.lockowner = me - return True + pass # already acquired by the current thread + else: + self.lock.acquire(True) + assert self.lockowner is None + assert self.lockcounter == 0 + self.lockowner = me + self.lockcounter += 1 def release_lock(self): - self.lockowner = None - self.lock.release() + me = self.space.getexecutioncontext() # used as thread ident + if self.lockowner is not me: + if not self._can_have_lock(): + return + space = self.space + raise OperationError(space.w_RuntimeError, + space.wrap("not holding the import lock")) + assert self.lockcounter > 0 + self.lockcounter -= 1 + if self.lockcounter == 0: + self.lockowner = None + self.lock.release() def getimportlock(space): return space.fromcache(ImportRLock) Modified: pypy/dist/pypy/module/thread/__init__.py ============================================================================== --- pypy/dist/pypy/module/thread/__init__.py (original) +++ pypy/dist/pypy/module/thread/__init__.py Sun Sep 16 12:54:23 2007 @@ -17,6 +17,11 @@ 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.getlocktype(space)', '_local': 'os_local.getlocaltype(space)', + + # custom interface for the 'imp' module + '_importlock_held': 'importlock.held', + '_importlock_acquire': 'importlock.acquire', + '_importlock_release': 'importlock.release', } def __init__(self, space, *args): Added: pypy/dist/pypy/module/thread/importlock.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/thread/importlock.py Sun Sep 16 12:54:23 2007 @@ -0,0 +1,14 @@ +# link between the app-level 'imp' module and '__builtin__.importing'. +# XXX maybe this should go together with 'imp.py' and the interp-level +# import logic into a MixedModule. + +from pypy.module.__builtin__ import importing + +def held(space): + return space.wrap(importing.getimportlock(space).lock_held()) + +def acquire(space): + importing.getimportlock(space).acquire_lock() + +def release(space): + importing.getimportlock(space).release_lock() Modified: pypy/dist/pypy/module/thread/test/test_import_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_import_lock.py (original) +++ pypy/dist/pypy/module/thread/test/test_import_lock.py Sun Sep 16 12:54:23 2007 @@ -10,12 +10,16 @@ cls.w_tmpdir = cls.space.wrap(tmpdir) def test_import_lock(self): - import thread + import thread, imp + assert not imp.lock_held() done = [] def f(): + print '[ENTER %d]' % i from imghdr import testall + print '[LEAVE %d]' % i done.append(1) for i in range(5): + print '[RUN %d]' % i thread.start_new_thread(f, ()) self.waitfor(lambda: len(done) == 5) assert len(done) == 5 @@ -24,23 +28,34 @@ import thread import re # -> causes nested imports - def test_no_lock_for_reimporting(self): - # CPython deadlocks in this situation; in our case the property - # of not requiring the import lock for already-imported modules - # is useful for translation, to avoid needing a prebuilt import - # lock object. - import os, sys - f = open(os.path.join(self.tmpdir, 'foobaz1.py'), 'w') - print >> f, """if 1: - import thread - lock = thread.allocate_lock() - lock.acquire() - def f(): - import sys - lock.release() - thread.start_new_thread(f, ()) - lock.acquire() - """ - f.close() - sys.path.insert(0, self.tmpdir) - import foobaz1 + def test_manual_locking(self): + import thread, os, imp, time, sys + f = open(os.path.join(self.tmpdir, 'foobaz2.py'), 'w') + f.close() # empty + done = [] + def f(): + sys.path.insert(0, self.tmpdir) + import foobaz2 + p = sys.path.pop(0) + assert p == self.tmpdir + done.append(1) + assert not imp.lock_held() + imp.acquire_lock() + assert imp.lock_held() + thread.start_new_thread(f, ()) + time.sleep(0.9) + assert not done + assert imp.lock_held() + # check that it's a recursive lock + imp.acquire_lock() + assert imp.lock_held() + imp.acquire_lock() + assert imp.lock_held() + imp.release_lock() + assert imp.lock_held() + imp.release_lock() + assert imp.lock_held() + imp.release_lock() + assert not imp.lock_held() + self.waitfor(lambda: done) + assert done From arigo at codespeak.net Sun Sep 16 13:18:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 13:18:56 +0200 (CEST) Subject: [pypy-svn] r46675 - pypy/dist/pypy/bin Message-ID: <20070916111856.8142680C7@code0.codespeak.net> Author: arigo Date: Sun Sep 16 13:18:55 2007 New Revision: 46675 Modified: pypy/dist/pypy/bin/py.py Log: Only import 'readline' when going interactive. This caused a failure in test_import.py, which got confused by escape characters printed by readline. For mysterious reasons this failure only started to occur today, and only on my laptop. Modified: pypy/dist/pypy/bin/py.py ============================================================================== --- pypy/dist/pypy/bin/py.py (original) +++ pypy/dist/pypy/bin/py.py Sun Sep 16 13:18:55 2007 @@ -122,6 +122,10 @@ # start the interactive console if go_interactive or getenv_w(space, 'PYTHONINSPECT'): + try: + import readline + except: + pass con = interactive.PyPyConsole( space, verbose=interactiveconfig.verbose, completer=interactiveconfig.completer) @@ -139,10 +143,6 @@ if __name__ == '__main__': - try: - import readline - except: - pass if hasattr(sys, 'setrecursionlimit'): # for running "python -i py.py -Si -- py.py -Si" sys.setrecursionlimit(3000) From arigo at codespeak.net Sun Sep 16 13:24:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 13:24:21 +0200 (CEST) Subject: [pypy-svn] r46676 - pypy/dist/pypy/tool Message-ID: <20070916112421.7920A80C4@code0.codespeak.net> Author: arigo Date: Sun Sep 16 13:24:21 2007 New Revision: 46676 Modified: pypy/dist/pypy/tool/cache.py Log: Allow this to be imported on top of pypy-c-NOthread too. Modified: pypy/dist/pypy/tool/cache.py ============================================================================== --- pypy/dist/pypy/tool/cache.py (original) +++ pypy/dist/pypy/tool/cache.py Sun Sep 16 13:24:21 2007 @@ -25,8 +25,11 @@ # Be sure to call the parent __init__() if you override it. # -from threading import RLock -lock = RLock() # multithreading protection +try: + from threading import RLock + lock = RLock() # multithreading protection +except ImportError: + lock = None class Cache(object): @@ -35,7 +38,7 @@ self._building = {} def getorbuild(self, key): - lock.acquire() + if lock: lock.acquire() try: try: return self.content[key] @@ -52,7 +55,7 @@ self._ready(result) return result finally: - lock.release() + if lock: lock.release() getorbuild._annspecialcase_ = "specialize:memo" def _ready(self, result): From tismer at codespeak.net Sun Sep 16 18:50:59 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 16 Sep 2007 18:50:59 +0200 (CEST) Subject: [pypy-svn] r46679 - pypy/dist/pypy/lib Message-ID: <20070916165059.E8B8780D3@code0.codespeak.net> Author: tismer Date: Sun Sep 16 18:50:59 2007 New Revision: 46679 Modified: pypy/dist/pypy/lib/stackless.py Log: fixed a bad bug from debugging. stackless emulation is working again. Sorry for all the confusion Modified: pypy/dist/pypy/lib/stackless.py ============================================================================== --- pypy/dist/pypy/lib/stackless.py (original) +++ pypy/dist/pypy/lib/stackless.py Sun Sep 16 18:50:59 2007 @@ -57,7 +57,7 @@ arguments *argl, **argd """ if self._frame is None or self._frame.dead: - self._frame = frame = MWrap(None)##GWrap() + self._frame = frame = GWrap() frame.coro = self if hasattr(self._frame, 'run') and self._frame.run: raise ValueError("cannot bind a bound coroutine") From arigo at codespeak.net Sun Sep 16 19:14:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 19:14:58 +0200 (CEST) Subject: [pypy-svn] r46680 - in pypy/dist/pypy/translator/c: src test Message-ID: <20070916171458.24B6E80DE@code0.codespeak.net> Author: arigo Date: Sun Sep 16 19:14:57 2007 New Revision: 46680 Modified: pypy/dist/pypy/translator/c/src/mem.h pypy/dist/pypy/translator/c/test/test_boehm.py Log: Run-time weakrefs to prebuilt objects... crash Boehm in ways that cost an afternoon of debugging :-( Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Sun Sep 16 19:14:57 2007 @@ -100,7 +100,12 @@ */ #define OP_CALL_BOEHM_GC_ALLOC(size, r) OP_BOEHM_ZERO_MALLOC(size, r, void *, 0, 0) -#define OP_BOEHM_DISAPPEARING_LINK(link, obj, r) \ +#define OP_BOEHM_DISAPPEARING_LINK(link, obj, r) \ + if (GC_base(obj) == NULL) \ + ; /* 'obj' is probably a prebuilt object - it makes no */ \ + /* sense to register it then, and it crashes Boehm in */ \ + /* quite obscure ways */ \ + else \ GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj) #endif /* USING_BOEHM_GC */ Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Sun Sep 16 19:14:57 2007 @@ -315,6 +315,21 @@ res = c_fn(0) assert res == -5 + def test_weakref_to_prebuilt(self): + import weakref + from pypy.rlib import rgc + class A: + pass + a = A() + a.hello = 42 + def fn(n): + lst = [weakref.ref(a) for i in range(n)] + rgc.collect() + for r in lst: + assert r() is a + c_fn = self.getcompiled(fn, [int]) + c_fn(100) + class TestUsingExactBoehm(TestUsingBoehm): gcpolicy = "exact_boehm" From arigo at codespeak.net Sun Sep 16 19:24:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 19:24:34 +0200 (CEST) Subject: [pypy-svn] r46681 - in pypy/dist/pypy: annotation translator/c/test Message-ID: <20070916172434.CB1D580E6@code0.codespeak.net> Author: arigo Date: Sun Sep 16 19:24:34 2007 New Revision: 46681 Modified: pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/translator/c/test/test_boehm.py Log: Require an annotation-time proof that we don't do weakref.ref(None). Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Sun Sep 16 19:24:34 2007 @@ -568,6 +568,9 @@ def weakref_ref(s_obj): if not isinstance(s_obj, SomeInstance): raise Exception("cannot take a weakref to %r" % (s_obj,)) + if s_obj.can_be_None: + raise Exception("should assert that the instance we take " + "a weakref to cannot be None") return SomeWeakRef(s_obj.classdef) BUILTIN_ANALYZERS[weakref.ref] = weakref_ref Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Sun Sep 16 19:24:34 2007 @@ -268,6 +268,7 @@ if i & 1 == 0: a = A() a.index = i + assert a is not None weakrefs.append(weakref.ref(a)) if i % 7 == 6: keepalive.append(a) From arigo at codespeak.net Sun Sep 16 19:26:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 19:26:03 +0200 (CEST) Subject: [pypy-svn] r46682 - pypy/dist/pypy/module/_weakref Message-ID: <20070916172603.B957880E6@code0.codespeak.net> Author: arigo Date: Sun Sep 16 19:26:03 2007 New Revision: 46682 Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py Log: TODOs in module/_weakref. Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Sun Sep 16 19:26:03 2007 @@ -70,11 +70,16 @@ class W_WeakrefBase(Wrappable): def __init__(w_self, space, w_obj, w_callable): + # XXX should use None instead of w_None for w_callable, + # because if this fails before w_self is fully initialized + # we get a segfault in the __del__ w_self.space = space w_self.w_obj_weak = weakref.ref(w_obj) w_self.w_callable = w_callable def dereference(self): + # XXX many callers would be simpler if this just returned None + # instead of w_None w_obj = self.w_obj_weak() if w_obj is None: return self.space.w_None From arigo at codespeak.net Sun Sep 16 19:45:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 19:45:40 +0200 (CEST) Subject: [pypy-svn] r46683 - pypy/dist/pypy/module/_weakref Message-ID: <20070916174540.2637480DD@code0.codespeak.net> Author: arigo Date: Sun Sep 16 19:45:39 2007 New Revision: 46683 Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py Log: Translation fix. Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Sun Sep 16 19:45:39 2007 @@ -74,6 +74,7 @@ # because if this fails before w_self is fully initialized # we get a segfault in the __del__ w_self.space = space + assert w_obj is not None w_self.w_obj_weak = weakref.ref(w_obj) w_self.w_callable = w_callable From cfbolz at codespeak.net Sun Sep 16 20:04:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 16 Sep 2007 20:04:03 +0200 (CEST) Subject: [pypy-svn] r46684 - pypy/dist/pypy/module/_weakref Message-ID: <20070916180403.5BA9880A2@code0.codespeak.net> Author: cfbolz Date: Sun Sep 16 20:04:02 2007 New Revision: 46684 Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py Log: fix the XXX Armin added Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Sun Sep 16 20:04:02 2007 @@ -70,24 +70,19 @@ class W_WeakrefBase(Wrappable): def __init__(w_self, space, w_obj, w_callable): - # XXX should use None instead of w_None for w_callable, - # because if this fails before w_self is fully initialized - # we get a segfault in the __del__ + if space.is_w(w_callable, space.w_None): + w_callable = None w_self.space = space assert w_obj is not None w_self.w_obj_weak = weakref.ref(w_obj) w_self.w_callable = w_callable def dereference(self): - # XXX many callers would be simpler if this just returned None - # instead of w_None w_obj = self.w_obj_weak() - if w_obj is None: - return self.space.w_None return w_obj - + def activate_callback(w_self): - if not w_self.space.is_w(w_self.w_callable, w_self.space.w_None): + if not w_self.w_callable is None: try: w_self.space.call_function(w_self.w_callable, w_self) except OperationError, e: @@ -103,12 +98,19 @@ if self.w_hash is not None: return self.w_hash w_obj = self.dereference() - if self.space.is_w(w_obj, self.space.w_None): + if w_obj is None: raise OperationError(self.space.w_TypeError, self.space.wrap("weak object has gone away")) self.w_hash = self.space.hash(w_obj) return self.w_hash + def descr_call(self): + w_obj = self.dereference() + if w_obj is None: + return self.space.w_None + return w_obj + + def descr__new__weakref(space, w_subtype, w_obj, w_callable=None): lifeline = w_obj.getweakref() if lifeline is None: @@ -119,8 +121,8 @@ def descr__eq__(space, ref1, ref2): w_obj1 = ref1.dereference() w_obj2 = ref2.dereference() - if (space.is_w(w_obj1, space.w_None) or - space.is_w(w_obj2, space.w_None)): + if (w_obj1 is None or + w_obj2 is None): return space.is_(ref1, ref2) return space.eq(w_obj1, w_obj2) @@ -137,7 +139,7 @@ __ne__ = interp2app(descr__ne__, unwrap_spec=[ObjSpace, W_Weakref, W_Weakref]), __hash__ = interp2app(W_Weakref.descr_hash, unwrap_spec=['self']), - __call__ = interp2app(W_Weakref.dereference, unwrap_spec=['self']) + __call__ = interp2app(W_Weakref.descr_call, unwrap_spec=['self']) ) @@ -204,8 +206,7 @@ if not isinstance(proxy, W_Proxy): return proxy w_obj = proxy.dereference() - assert w_obj is not None - if space.is_w(w_obj, space.w_None): + if w_obj is None: raise OperationError( space.w_ReferenceError, space.wrap("weakly referenced object no longer exists")) From arigo at codespeak.net Sun Sep 16 21:18:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 21:18:09 +0200 (CEST) Subject: [pypy-svn] r46685 - pypy/dist/pypy/tool Message-ID: <20070916191809.7922780E0@code0.codespeak.net> Author: arigo Date: Sun Sep 16 21:18:08 2007 New Revision: 46685 Modified: pypy/dist/pypy/tool/error.py Log: Don't crash if FunctionDesc.pyobj is None. Modified: pypy/dist/pypy/tool/error.py ============================================================================== --- pypy/dist/pypy/tool/error.py (original) +++ pypy/dist/pypy/tool/error.py Sun Sep 16 21:18:08 2007 @@ -117,8 +117,12 @@ descs = annotator.bindings[oper.args[0]].descriptions for desc in descs.keys(): func = desc.pyobj - msg.append(" function %s <%s, line %s> returning" % (func.func_name, - func.func_code.co_filename, func.func_code.co_firstlineno)) + if func is None: + r = repr(desc) + else: + r = "function %s <%s, line %s>" % (func.func_name, + func.func_code.co_filename, func.func_code.co_firstlineno) + msg.append(" %s returning" % (r,)) graph = desc.getuniquegraph() msg.append(" %s" % annotator.binding( graph.returnblock.inputargs[0], "(no annotation)")) From arigo at codespeak.net Sun Sep 16 21:27:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Sep 2007 21:27:11 +0200 (CEST) Subject: [pypy-svn] r46686 - pypy/dist/pypy/tool Message-ID: <20070916192711.1009380DE@code0.codespeak.net> Author: arigo Date: Sun Sep 16 21:27:10 2007 New Revision: 46686 Modified: pypy/dist/pypy/tool/error.py Log: More of the same. Modified: pypy/dist/pypy/tool/error.py ============================================================================== --- pypy/dist/pypy/tool/error.py (original) +++ pypy/dist/pypy/tool/error.py Sun Sep 16 21:27:10 2007 @@ -123,9 +123,13 @@ r = "function %s <%s, line %s>" % (func.func_name, func.func_code.co_filename, func.func_code.co_firstlineno) msg.append(" %s returning" % (r,)) - graph = desc.getuniquegraph() - msg.append(" %s" % annotator.binding( - graph.returnblock.inputargs[0], "(no annotation)")) + if hasattr(desc, 'getuniquegraph'): + graph = desc.getuniquegraph() + r = annotator.binding(graph.returnblock.inputargs[0], + "(no annotation)") + else: + r = '?' + msg.append(" %s" % (r,)) msg.append("") def format_someobject_error(annotator, position_key, what, s_value, called_from_graph, binding=""): From exarkun at codespeak.net Sun Sep 16 21:55:58 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sun, 16 Sep 2007 21:55:58 +0200 (CEST) Subject: [pypy-svn] r46687 - pypy/branch/zlib-rffi-impl/pypy/module/zlib Message-ID: <20070916195558.A320480ED@code0.codespeak.net> Author: exarkun Date: Sun Sep 16 21:55:56 2007 New Revision: 46687 Modified: pypy/branch/zlib-rffi-impl/pypy/module/zlib/__init__.py pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py Log: try to simplify (by adding bugs) to get it to work (no dice); replace _hints hack at least, and give __new__s unwrap_specs Modified: pypy/branch/zlib-rffi-impl/pypy/module/zlib/__init__.py ============================================================================== --- pypy/branch/zlib-rffi-impl/pypy/module/zlib/__init__.py (original) +++ pypy/branch/zlib-rffi-impl/pypy/module/zlib/__init__.py Sun Sep 16 21:55:56 2007 @@ -3,13 +3,8 @@ Mixed-module definition for the zlib module. """ -import zlib - from pypy.interpreter.mixedmodule import MixedModule -def constant(value): - return 'space.wrap(%s)' % (value,) - class Module(MixedModule): interpleveldefs = { @@ -19,16 +14,6 @@ 'Decompress': 'interp_zlib.Decompress', } -# # Constants exposed by zlib.h -# interpleveldefs.update(( -# (name, constant(getattr(zlib, name))) -# for name -# in ['DEFLATED', 'DEF_MEM_LEVEL', 'MAX_WBITS', -# 'Z_BEST_COMPRESSION', 'Z_BEST_SPEED', -# 'Z_DEFAULT_COMPRESSION', 'Z_DEFAULT_STRATEGY', -# 'Z_FILTERED', 'Z_FINISH', 'Z_FULL_FLUSH', -# 'Z_HUFFMAN_ONLY', 'Z_NO_FLUSH', 'Z_SYNC_FLUSH'])) - appleveldefs = { 'error': 'app_zlib.error', 'compress': 'app_zlib.compress', Modified: pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py (original) +++ pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py Sun Sep 16 21:55:56 2007 @@ -1,11 +1,11 @@ # import zlib -from pypy.interpreter.gateway import ObjSpace, interp2app +from pypy.interpreter.gateway import ObjSpace, W_Root, interp2app from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.error import OperationError -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import rffi, lltype, llmemory from pypy.rpython.tool import rffi_platform includes = ['zlib.h'] @@ -28,6 +28,9 @@ Z_STREAM_ERROR = rffi_platform.ConstantInteger('Z_STREAM_ERROR') ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION') + + Z_DEFLATED = rffi_platform.ConstantInteger('Z_DEFLATED') + Z_DEFAULT_STRATEGY = rffi_platform.ConstantInteger('Z_DEFAULT_STRATEGY') Z_DEFAULT_COMPRESSION = rffi_platform.ConstantInteger( 'Z_DEFAULT_COMPRESSION') Z_NO_FLUSH = rffi_platform.ConstantInteger( @@ -44,8 +47,11 @@ Z_STREAM_ERROR = config['Z_STREAM_ERROR'] ZLIB_VERSION = config['ZLIB_VERSION'] + Z_DEFAULT_COMPRESSION = config['Z_DEFAULT_COMPRESSION'] +Z_DEFAULT_STRATEGY = config['Z_DEFAULT_STRATEGY'] Z_NO_FLUSH = config['Z_DEFAULT_COMPRESSION'] +Z_DEFLATED = config['Z_DEFLATED'] class ComplexCConfig: """ @@ -89,11 +95,31 @@ _crc32 = zlib_external('crc32', [uLong, Bytefp, uInt], uLong) _adler32 = zlib_external('adler32', [uLong, Bytefp, uInt], uLong) -_deflateInit = zlib_external( - 'deflateInit_', [z_stream_p, rffi.INT, rffi.CCHARP, rffi.INT], + + +# XXX I want to call deflateInit2, not deflateInit2_ +_deflateInit2_ = zlib_external( + 'deflateInit2_', + [z_stream_p, # stream + rffi.INT, # level + rffi.INT, # method + rffi.INT, # window bits + rffi.INT, # mem level + rffi.INT, # strategy + rffi.CCHARP, # version + rffi.INT], # stream size rffi.INT) _deflate = zlib_external('deflate', [z_stream_p, rffi.INT], rffi.INT) +def _deflateInit2(stream, level, method, wbits, memlevel, strategy): + version = rffi.str2charp(ZLIB_VERSION) + size = llmemory.sizeof(z_stream) + result = _deflateInit2_( + stream, level, method, wbits, memlevel, strategy, version, size) + rffi.free_charp(version) + return result + + def crc32(space, string, start=0): """ crc32(string[, start]) -- Compute a CRC-32 checksum of string. @@ -143,6 +169,7 @@ self.stream.c_zalloc = lltype.nullptr(z_stream.c_zalloc.TO) self.stream.c_zfree = lltype.nullptr(z_stream.c_zfree.TO) self.stream.c_avail_in = rffi.cast(lltype.Unsigned, 0) + self.stream.c_next_in = lltype.nullptr(z_stream.c_next_in.TO) def __del__(self): @@ -173,11 +200,13 @@ _StreamBase.__init__(self, space) - # XXX I want to call deflateInit, not deflateInit_ - size = z_stream._hints['size'] - version = rffi.str2charp(ZLIB_VERSION) - result = _deflateInit(self.stream, level, version, size) - rffi.free_charp(version) + method = Z_DEFLATED + windowBits = 15 + memLevel = 8 + strategy = Z_DEFAULT_STRATEGY + + result = _deflateInit2( + self.stream, level, method, windowBits, memLevel, strategy) if result != Z_OK: raise error_from_zlib(self.space, result) @@ -197,22 +226,13 @@ Bytefp.TO, len(data), flavor='raw') for i in xrange(len(data)): self.stream.c_next_in[i] = rffi.cast(Bytef, data[i]) - try: - self.stream.c_avail_out = rffi.cast(lltype.Unsigned, length) - self.stream.c_next_out = lltype.malloc( - Bytefp.TO, length, flavor='raw') - try: - result = _deflate(self.stream, Z_NO_FLUSH) - if result != Z_OK: - raise error_from_zlib(self.space, result) - else: - return rffi.charp2str(self.stream.c_next_out) - finally: - lltype.free(self.stream.c_next_out, flavor='raw') - self.stream.c_avail_out = rffi.cast(lltype.Unsigned, 0) - finally: - lltype.free(self.stream.c_next_in, flavor='raw') - self.stream.c_avail_in = rffi.cast(lltype.Unsigned, 0) + self.stream.c_avail_out = rffi.cast(lltype.Unsigned, length) + self.stream.c_next_out = lltype.malloc( + Bytefp.TO, length, flavor='raw') + result = _deflate(self.stream, Z_NO_FLUSH) + if result != Z_OK: + raise error_from_zlib(self.space, result) + return rffi.charp2str(self.stream.c_next_out) compress.unwrap_spec = ['self', str, int] @@ -232,14 +252,15 @@ flush.unwrap_spec = ['self', int] -def Compress___new__(space, w_subtype, w_anything=None): +def Compress___new__(space, w_subtype, w_level=None): """ Create a new z_stream and call its initializer. """ stream = space.allocate_instance(Compress, w_subtype) stream = space.interp_w(Compress, stream) - Compress.__init__(stream, space, w_anything) + Compress.__init__(stream, space, w_level) return space.wrap(stream) +Compress___new__.unwrap_spec = [ObjSpace, W_Root, W_Root] Compress.typedef = TypeDef( @@ -306,7 +327,7 @@ stream = space.interp_w(Decompress, stream) Decompress.__init__(stream, space, w_anything) return space.wrap(stream) - +Decompress___new__.unwrap_spec = [ObjSpace, W_Root, W_Root] Decompress.typedef = TypeDef( From arigo at codespeak.net Mon Sep 17 10:31:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Sep 2007 10:31:24 +0200 (CEST) Subject: [pypy-svn] r46688 - pypy/dist/pypy/translator/c/test Message-ID: <20070917083124.5E51580F9@code0.codespeak.net> Author: arigo Date: Mon Sep 17 10:31:17 2007 New Revision: 46688 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: Annotation fix. Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Mon Sep 17 10:31:17 2007 @@ -582,6 +582,7 @@ if i & 1 == 0: a = A() a.index = i + assert a is not None weakrefs.append(weakref.ref(a)) if i % 7 == 6: keepalive.append(a) From arigo at codespeak.net Mon Sep 17 10:39:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Sep 2007 10:39:00 +0200 (CEST) Subject: [pypy-svn] r46689 - pypy/dist/pypy/lib/test2 Message-ID: <20070917083900.8C69580F9@code0.codespeak.net> Author: arigo Date: Mon Sep 17 10:39:00 2007 New Revision: 46689 Modified: pypy/dist/pypy/lib/test2/test_stackless.py Log: Skip the end of this test on top of py.py. If I get it right it has no chance to pass there. Modified: pypy/dist/pypy/lib/test2/test_stackless.py ============================================================================== --- pypy/dist/pypy/lib/test2/test_stackless.py (original) +++ pypy/dist/pypy/lib/test2/test_stackless.py Mon Sep 17 10:39:00 2007 @@ -1,16 +1,20 @@ -from pypy.conftest import gettestobjspace +from pypy.conftest import gettestobjspace, option class AppTest_Stackless: def setup_class(cls): space = gettestobjspace(usemodules=('_stackless',)) cls.space = space + # cannot test the unpickle part on top of py.py + cls.w_can_unpickle = space.wrap(bool(option.runappdirect)) def test_pickle(self): import new, sys mod = new.module('mod') sys.modules['mod'] = mod + mod.can_unpickle = self.can_unpickle + mod.skip = skip try: exec ''' import pickle, sys @@ -39,6 +43,8 @@ t = stackless.tasklet(demo)(lev) stackless.run() assert seen == range(1, lev+1) + range(lev, 0, -1) +if not can_unpickle: + skip("cannot test the unpickling part on top of py.py") print "now running the clone" tt = pickle.loads(blob) tt.insert() From arigo at codespeak.net Mon Sep 17 11:21:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Sep 2007 11:21:22 +0200 (CEST) Subject: [pypy-svn] r46690 - pypy/dist/pypy/interpreter/pyparser Message-ID: <20070917092122.6FBFA80E3@code0.codespeak.net> Author: arigo Date: Mon Sep 17 11:21:21 2007 New Revision: 46690 Modified: pypy/dist/pypy/interpreter/pyparser/pythonlexer.py Log: Fix. Modified: pypy/dist/pypy/interpreter/pyparser/pythonlexer.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/pythonlexer.py (original) +++ pypy/dist/pypy/interpreter/pyparser/pythonlexer.py Mon Sep 17 11:21:21 2007 @@ -371,7 +371,7 @@ def get_source_text(self, p0, p1): "We get passed two token stack positions." - return "XXX this got left behind in a refactoring. Stack positions are %d and %d" % (p1, p2) + return "XXX this got left behind in a refactoring. Stack positions are %d and %d" % (p0, p1) def debug(self): """return context for debug information""" From antocuni at codespeak.net Mon Sep 17 14:44:21 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 17 Sep 2007 14:44:21 +0200 (CEST) Subject: [pypy-svn] r46693 - in pypy/dist/pypy/translator: cli js jvm oosupport Message-ID: <20070917124421.61EB180EC@code0.codespeak.net> Author: antocuni Date: Mon Sep 17 14:44:19 2007 New Revision: 46693 Modified: pypy/dist/pypy/translator/cli/function.py pypy/dist/pypy/translator/cli/gencli.py pypy/dist/pypy/translator/js/function.py pypy/dist/pypy/translator/js/js.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/oosupport/function.py pypy/dist/pypy/translator/oosupport/genoo.py Log: prepare genjvm to use stack_optimizations, though it's disabled right now. Move the smart version of render_bool_switch from cli to oosupport, so genjvm uses it. Move the old non-smart version to js. Modified: pypy/dist/pypy/translator/cli/function.py ============================================================================== --- pypy/dist/pypy/translator/cli/function.py (original) +++ pypy/dist/pypy/translator/cli/function.py Mon Sep 17 14:44:19 2007 @@ -1,13 +1,8 @@ -try: - set -except NameError: - from sets import Set as set - from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem.lltype import Void from pypy.rpython.ootypesystem import ootype from pypy.translator.oosupport.treebuilder import SubOperation -from pypy.translator.oosupport.function import Function as OOFunction +from pypy.translator.oosupport.function import Function as OOFunction, render_sub_op from pypy.translator.oosupport.constant import push_constant from pypy.translator.cli.option import getoption from pypy.translator.cli.cts import CTS @@ -142,23 +137,6 @@ self.store(link.last_exc_value) self._setup_link(link) - # XXX: this method should be moved into oosupport, but other - # backends are not ready :-( - def render_bool_switch(self, block): - assert len(block.exits) == 2 - for link in block.exits: - if link.exitcase: - link_true = link - else: - link_false = link - - true_label = self.next_label('link_true') - self.generator.load(block.exitswitch) - self.generator.branch_conditionally(True, true_label) - self._follow_link(link_false) # if here, the exitswitch is false - self.set_label(true_label) - self._follow_link(link_true) # if here, the exitswitch is true - def render_numeric_switch(self, block): if block.exitswitch.concretetype in (ootype.SignedLongLong, ootype.UnsignedLongLong): # TODO: it could be faster to check is the values fit in @@ -200,7 +178,7 @@ else: self.ilasm.load_local(v) elif isinstance(v, SubOperation): - self._render_sub_op(v) + render_sub_op(v, self.db, self.generator) else: super(Function, self).load(v) Modified: pypy/dist/pypy/translator/cli/gencli.py ============================================================================== --- pypy/dist/pypy/translator/cli/gencli.py (original) +++ pypy/dist/pypy/translator/cli/gencli.py Mon Sep 17 14:44:19 2007 @@ -5,8 +5,6 @@ from py.compat import subprocess from pypy.config.config import Config from pypy.translator.oosupport.genoo import GenOO -from pypy.translator.oosupport.treebuilder import build_trees -from pypy.translator.backendopt.ssa import SSI_to_SSA from pypy.translator.cli import conftest from pypy.translator.cli.ilgenerator import IlasmGenerator from pypy.translator.cli.function import Function, log @@ -22,11 +20,6 @@ from pypy.translator.cli import query from pypy.translator.cli import constant -try: - set -except NameError: - from sets import Set as set - class GenCli(GenOO): TypeSystem = CTS Function = Function @@ -45,26 +38,14 @@ WeakRefConst = constant.CLIWeakRefConst def __init__(self, tmpdir, translator, entrypoint, config=None, exctrans=False): - GenOO.__init__(self, tmpdir, translator, entrypoint, config) - exctrans = exctrans or translator.config.translation.cli.exception_transformer - if exctrans: - self.db.exceptiontransformer = translator.getexceptiontransformer() - - for node in get_prebuilt_nodes(translator, self.db): - self.db.pending_node(node) + GenOO.__init__(self, tmpdir, translator, entrypoint, config, exctrans) self.assembly_name = entrypoint.get_name() self.tmpfile = tmpdir.join(self.assembly_name + '.il') self.const_stat = str(tmpdir.join('const_stat')) - if exctrans: - etrafo = self.db.exceptiontransformer - for graph in translator.graphs: - etrafo.create_exception_handling(graph) - - if translator.config.translation.backendopt.stack_optimization: - for graph in translator.graphs: - SSI_to_SSA(graph) - build_trees(graph) + def append_prebuilt_nodes(self): + for node in get_prebuilt_nodes(self.translator, self.db): + self.db.pending_node(node) def generate_source(self): GenOO.generate_source(self) Modified: pypy/dist/pypy/translator/js/function.py ============================================================================== --- pypy/dist/pypy/translator/js/function.py (original) +++ pypy/dist/pypy/translator/js/function.py Mon Sep 17 14:44:19 2007 @@ -201,6 +201,19 @@ self.ilasm.catch() #self.ilasm.close_branch() + # XXX: soon or later we should use the smarter version in oosupport + def render_bool_switch(self, block): + for link in block.exits: + self._setup_link(link) + target_label = self._get_block_name(link.target) + if link is block.exits[-1]: + self.generator.branch_unconditionally(target_label) + else: + assert type(link.exitcase) is bool + assert block.exitswitch is not None + self.generator.load(block.exitswitch) + self.generator.branch_conditionally(link.exitcase, target_label) + def record_ll_meta_exc(self, ll_meta_exc): pass Modified: pypy/dist/pypy/translator/js/js.py ============================================================================== --- pypy/dist/pypy/translator/js/js.py (original) +++ pypy/dist/pypy/translator/js/js.py Mon Sep 17 14:44:19 2007 @@ -66,7 +66,10 @@ self.use_debug = use_debug self.assembly_name = self.translator.graphs[0].name self.tmpfile = udir.join(self.assembly_name + '.js') - + + def stack_optimization(self): + pass # genjs does not support treebuilder + def gen_pendings(self): while self.db._pending_nodes: node = self.db._pending_nodes.pop() Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Mon Sep 17 14:44:19 2007 @@ -1,5 +1,7 @@ from pypy.objspace.flow import model as flowmodel from pypy.translator.oosupport.metavm import Generator +from pypy.translator.oosupport.treebuilder import SubOperation +from pypy.translator.oosupport.function import render_sub_op from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.translator.oosupport.constant import push_constant @@ -941,6 +943,9 @@ jty, idx = self._var_data(value) return self.load_jvm_var(jty, idx) + if isinstance(value, SubOperation): + return render_sub_op(value, self.db, self) + if isinstance(value, flowmodel.Constant): return push_constant(self.db, value.concretetype, value.value, self) @@ -1365,7 +1370,7 @@ return str(arg) strargs = [jasmin_syntax(arg) for arg in args] instr_text = '%s %s' % (jvmstr, " ".join(strargs)) - #self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) + self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) self.curclass.out(' %-60s\n' % (instr_text,)) self.curfunc.instr_counter+=1 Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Mon Sep 17 14:44:19 2007 @@ -262,9 +262,14 @@ 'entrypoint' --- if supplied, an object with a render method """ GenOO.__init__(self, tmpdir, translator, entrypoint) - create_interlink_node(self.db) self.jvmsrc = JvmGeneratedSource(tmpdir, getoption('package')) + def append_prebuilt_nodes(self): + create_interlink_node(self.db) + + def stack_optimization(self): + pass # TODO: enable stack_optimization + def generate_source(self): """ Creates the sources, and returns a JvmGeneratedSource object for manipulating them """ Modified: pypy/dist/pypy/translator/oosupport/function.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/function.py (original) +++ pypy/dist/pypy/translator/oosupport/function.py Mon Sep 17 14:44:19 2007 @@ -8,7 +8,21 @@ from pypy.translator.oosupport.treebuilder import SubOperation from pypy.translator.oosupport.metavm import InstructionList, StoreResult - +def render_sub_op(sub_op, db, generator): + op = sub_op.op + instr_list = db.genoo.opcodes.get(op.opname, None) + assert instr_list is not None, 'Unknown opcode: %s ' % op + assert isinstance(instr_list, InstructionList) + assert instr_list[-1] is StoreResult, "Cannot inline an operation that doesn't store the result" + + # record that we know about the type of result and args + db.cts.lltype_to_cts(op.result.concretetype) + for v in op.args: + db.cts.lltype_to_cts(v.concretetype) + + instr_list = InstructionList(instr_list[:-1]) # leave the value on the stack if this is a sub-op + instr_list.render(generator, op) + # now the value is on the stack class Function(object): @@ -209,19 +223,20 @@ else: assert False, 'Unknonw exitswitch type: %s' % block.exitswitch.concretetype - # XXX: soon or later we should use the implementation in - # cli/function.py, but at the moment jvm and js fail with it. def render_bool_switch(self, block): + assert len(block.exits) == 2 for link in block.exits: - self._setup_link(link) - target_label = self._get_block_name(link.target) - if link is block.exits[-1]: - self.generator.branch_unconditionally(target_label) + if link.exitcase: + link_true = link else: - assert type(link.exitcase) is bool - assert block.exitswitch is not None - self.generator.load(block.exitswitch) - self.generator.branch_conditionally(link.exitcase, target_label) + link_false = link + + true_label = self.next_label('link_true') + self.generator.load(block.exitswitch) + self.generator.branch_conditionally(True, true_label) + self._follow_link(link_false) # if here, the exitswitch is false + self.set_label(true_label) + self._follow_link(link_true) # if here, the exitswitch is true def render_numeric_switch(self, block): log.WARNING("The default version of render_numeric_switch is *slow*: please override it in the backend") @@ -329,22 +344,6 @@ if self._trace_enabled(): self._trace_value('Result', op.result) - def _render_sub_op(self, sub_op): - op = sub_op.op - instr_list = self.db.genoo.opcodes.get(op.opname, None) - assert instr_list is not None, 'Unknown opcode: %s ' % op - assert isinstance(instr_list, InstructionList) - assert instr_list[-1] is StoreResult, "Cannot inline an operation that doesn't store the result" - - # record that we know about the type of result and args - self.cts.lltype_to_cts(op.result.concretetype) - for v in op.args: - self.cts.lltype_to_cts(v.concretetype) - - instr_list = InstructionList(instr_list[:-1]) # leave the value on the stack if this is a sub-op - instr_list.render(self.generator, op) - # now the value is on the stack - # ---------------------------------------------------------# # These methods are quite backend independent, but not # # used in all backends. Invoke them from your __init__ if # Modified: pypy/dist/pypy/translator/oosupport/genoo.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/genoo.py (original) +++ pypy/dist/pypy/translator/oosupport/genoo.py Mon Sep 17 14:44:19 2007 @@ -1,6 +1,10 @@ """ basic oogenerator """ + +from py.builtin import set from pypy.translator.oosupport import constant as ooconst +from pypy.translator.oosupport.treebuilder import build_trees +from pypy.translator.backendopt.ssa import SSI_to_SSA class GenOO(object): TypeSystem = None @@ -23,7 +27,7 @@ DictConst = ooconst.DictConst WeakRefConst = ooconst.WeakRefConst - def __init__(self, tmpdir, translator, entrypoint, config=None): + def __init__(self, tmpdir, translator, entrypoint, config=None, exctrans=False): self.tmpdir = tmpdir self.translator = translator self.entrypoint = entrypoint @@ -33,6 +37,29 @@ config = get_pypy_config(translating=True) self.config = config + # XXX: move this option out of the 'cli' section + exctrans = exctrans or translator.config.translation.cli.exception_transformer + if exctrans: + self.db.exceptiontransformer = translator.getexceptiontransformer() + + self.append_prebuilt_nodes() + + if exctrans: + etrafo = self.db.exceptiontransformer + for graph in translator.graphs: + etrafo.create_exception_handling(graph) + + if translator.config.translation.backendopt.stack_optimization: + self.stack_optimization() + + def stack_optimization(self): + for graph in self.translator.graphs: + SSI_to_SSA(graph) + build_trees(graph) + + def append_prebuilt_nodes(self): + pass + def generate_source(self): self.ilasm = self.create_assembler() self.fix_names() From antocuni at codespeak.net Mon Sep 17 15:32:28 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 17 Sep 2007 15:32:28 +0200 (CEST) Subject: [pypy-svn] r46694 - pypy/dist/pypy/translator/jvm Message-ID: <20070917133228.996E880FC@code0.codespeak.net> Author: antocuni Date: Mon Sep 17 15:32:22 2007 New Revision: 46694 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py Log: enable stack_optimizations for jvm; to do this, we must compare Variables by name and not by identity. Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Mon Sep 17 15:32:22 2007 @@ -515,14 +515,14 @@ idx = self.next_offset self.next_offset += jtype.descriptor.type_width() if jvar: - assert jvar not in self.local_vars # never been added before - self.local_vars[jvar] = idx + assert jvar.name not in self.local_vars # never been added before + self.local_vars[jvar.name] = idx self.function_arguments.append((jtype, idx)) return idx def var_offset(self, jvar, jtype): """ Returns offset for variable 'jvar', of java type 'jtype' """ - if jvar in self.local_vars: - return self.local_vars[jvar] + if jvar.name in self.local_vars: + return self.local_vars[jvar.name] return self.add_var(jvar, jtype) @@ -1370,7 +1370,7 @@ return str(arg) strargs = [jasmin_syntax(arg) for arg in args] instr_text = '%s %s' % (jvmstr, " ".join(strargs)) - self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) + #self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) self.curclass.out(' %-60s\n' % (instr_text,)) self.curfunc.instr_counter+=1 Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Mon Sep 17 15:32:22 2007 @@ -267,9 +267,6 @@ def append_prebuilt_nodes(self): create_interlink_node(self.db) - def stack_optimization(self): - pass # TODO: enable stack_optimization - def generate_source(self): """ Creates the sources, and returns a JvmGeneratedSource object for manipulating them """ From arigo at codespeak.net Mon Sep 17 18:31:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Sep 2007 18:31:00 +0200 (CEST) Subject: [pypy-svn] r46699 - pypy/dist/pypy/interpreter Message-ID: <20070917163100.290C98115@code0.codespeak.net> Author: arigo Date: Mon Sep 17 18:30:59 2007 New Revision: 46699 Modified: pypy/dist/pypy/interpreter/baseobjspace.py Log: Avoid importing pypy.module.thread if unnecessary (the import crashes on non-Linux platforms). Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Mon Sep 17 18:30:59 2007 @@ -438,15 +438,20 @@ """Return an interp-level Lock object if threads are enabled, and a dummy object if they are not.""" if self.config.objspace.usemodules.thread: - from pypy.module.thread.ll_thread import allocate_lock, error - try: - return allocate_lock() - except error: - raise OperationError(self.w_RuntimeError, - self.wrap("out of resources")) + # we use a sub-function to avoid putting the 'import' statement + # here, where the flow space would see it even if thread=False + return self.__allocate_lock() else: return dummy_lock + def __allocate_lock(self): + from pypy.module.thread.ll_thread import allocate_lock, error + try: + return allocate_lock() + except error: + raise OperationError(self.w_RuntimeError, + self.wrap("out of resources")) + # Following is a friendly interface to common object space operations # that can be defined in term of more primitive ones. Subclasses # may also override specific functions for performance. From simonb at codespeak.net Tue Sep 18 04:32:16 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Tue, 18 Sep 2007 04:32:16 +0200 (CEST) Subject: [pypy-svn] r46702 - pypy/dist/pypy/rpython/numpy Message-ID: <20070918023216.85294812F@code0.codespeak.net> Author: simonb Date: Tue Sep 18 04:32:15 2007 New Revision: 46702 Modified: pypy/dist/pypy/rpython/numpy/rarray.py Log: rtyping of the copy method Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Tue Sep 18 04:32:15 2007 @@ -284,6 +284,23 @@ cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) return hop.gendirectcall(ll_transpose, cARRAY, v_self) + def rtype_method_copy(self, hop): + # This is very similar to astype method. Could factor something out perhaps. + r_result = hop.r_result + [v_self] = hop.inputargs(self) + cARRAY = hop.inputconst(Void, hop.r_result.ARRAY.TO) + v_result = hop.llops.gendirectcall(ll_build_like, cARRAY, v_self) + # XX if we know v_self is contiguous, we should just do a memcopy XX + iter_new, iter_broadcast = gen_iter_funcs(self.ndim) + cbroadcast = hop.inputconst(Void, iter_broadcast) + cITER0 = hop.inputconst(Void, r_result.ITER.TO) + v_it0 = hop.gendirectcall(iter_new, cITER0, v_result, v_result, cbroadcast) + cITER1 = hop.inputconst(Void, self.ITER.TO) + v_it1 = hop.gendirectcall(iter_new, cITER1, v_self, v_self, cbroadcast) + cITEM = hop.inputconst(Void, r_result.ITEM) + hop.gendirectcall(ll_array_set, cITEM, v_it0, v_it1) + return v_result + def rtype_method_reshape(self, hop): r_result = hop.r_result r_tuple = hop.args_r[1] @@ -769,3 +786,5 @@ + + From arigo at codespeak.net Tue Sep 18 09:56:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 09:56:09 +0200 (CEST) Subject: [pypy-svn] r46704 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070918075609.CFDD6812F@code0.codespeak.net> Author: arigo Date: Tue Sep 18 09:56:08 2007 New Revision: 46704 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Support for recursive types in ll2ctypes. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Tue Sep 18 09:56:08 2007 @@ -21,6 +21,8 @@ _ctypes_cache = {} +_delayed_ptrs = [] +_gettype_recursion = 0 def _setup_ctypes_cache(): from pypy.rpython.lltypesystem import rffi @@ -129,32 +131,58 @@ try: return _ctypes_cache[T] except KeyError: - if isinstance(T, lltype.Ptr): - if isinstance(T.TO, lltype.FuncType): - argtypes = [get_ctypes_type(ARG) for ARG in T.TO.ARGS] - if T.TO.RESULT is lltype.Void: - restype = None - else: - restype = get_ctypes_type(T.TO.RESULT) - cls = ctypes.CFUNCTYPE(restype, *argtypes) + global _gettype_recursion + _gettype_recursion += 1 + try: + cls = build_new_ctypes_type(T) + if T not in _ctypes_cache: + _ctypes_cache[T] = cls else: - cls = ctypes.POINTER(get_ctypes_type(T.TO)) - elif isinstance(T, lltype.Struct): - cls = build_ctypes_struct(T) - elif isinstance(T, lltype.Array): - cls = build_ctypes_array(T) - elif isinstance(T, lltype.OpaqueType): - if T.hints.get('external', None) != 'C': - raise TypeError("%s is not external" % T) - cls = ctypes.c_char * T.hints['getsize']() - else: - _setup_ctypes_cache() - if T in _ctypes_cache: - return _ctypes_cache[T] - raise NotImplementedError(T) - _ctypes_cache[T] = cls + # check for buggy recursive structure logic + assert _ctypes_cache[T] is cls + + if _gettype_recursion == 1: + complete_pointer_types() + finally: + _gettype_recursion -= 1 return cls +def build_new_ctypes_type(T): + if isinstance(T, lltype.Ptr): + if isinstance(T.TO, lltype.FuncType): + argtypes = [get_ctypes_type(ARG) for ARG in T.TO.ARGS] + if T.TO.RESULT is lltype.Void: + restype = None + else: + restype = get_ctypes_type(T.TO.RESULT) + return ctypes.CFUNCTYPE(restype, *argtypes) + elif isinstance(T.TO, lltype.Struct): + # for recursive structures: build a forward pointer first + uniquename = 'ctypes_%s_%d' % (T.TO.__name__, len(_ctypes_cache)) + pcls = ctypes.POINTER(uniquename) + _delayed_ptrs.append((pcls, T.TO)) + return pcls + else: + return ctypes.POINTER(get_ctypes_type(T.TO)) + elif isinstance(T, lltype.Struct): + return build_ctypes_struct(T) + elif isinstance(T, lltype.Array): + return build_ctypes_array(T) + elif isinstance(T, lltype.OpaqueType): + if T.hints.get('external', None) != 'C': + raise TypeError("%s is not external" % T) + return ctypes.c_char * T.hints['getsize']() + else: + _setup_ctypes_cache() + if T in _ctypes_cache: + return _ctypes_cache[T] + raise NotImplementedError(T) + +def complete_pointer_types(): + while _delayed_ptrs: + pcls, S = _delayed_ptrs.pop() + ctypes.SetPointerType(pcls, get_ctypes_type(S)) + def convert_struct(container, cstruct=None): STRUCT = container._TYPE Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Tue Sep 18 09:56:08 2007 @@ -51,6 +51,28 @@ assert sc.contents.y == 52 lltype.free(s, flavor='raw') + def test_struct_ptrs(self): + S2 = lltype.Struct('S2', ('y', lltype.Signed)) + S1 = lltype.Struct('S', ('x', lltype.Signed), ('p', lltype.Ptr(S2))) + s1 = lltype.malloc(S1, flavor='raw') + s2a = lltype.malloc(S2, flavor='raw') + s2b = lltype.malloc(S2, flavor='raw') + s2a.y = ord('a') + s2b.y = ord('b') + sc1 = lltype2ctypes(s1) + sc1.contents.x = 50 + assert s1.x == 50 + sc1.contents.p = lltype2ctypes(s2a) + assert s1.p == s2a + s1.p.y -= 32 + assert sc1.contents.p.contents.y == ord('A') + s1.p = s2b + sc1.contents.p.contents.y -= 32 + assert s2b.y == ord('B') + lltype.free(s1, flavor='raw') + lltype.free(s2a, flavor='raw') + lltype.free(s2b, flavor='raw') + def test_simple_array(self): A = lltype.Array(lltype.Signed) a = lltype.malloc(A, 10, flavor='raw') @@ -406,9 +428,78 @@ lltype.free(s, flavor='raw') def test_recursive_struct(self): - py.test.skip("Not implemented") SX = lltype.ForwardReference() - S1 = lltype.Struct('S1', ('x', lltype.Ptr(SX))) - S1.x.TO.become(S1) - s = lltype.malloc(S1, flavor='raw') - sc = lltype2ctypes(s) + S1 = lltype.Struct('S1', ('p', lltype.Ptr(SX)), ('x', lltype.Signed)) + SX.become(S1) + # a chained list + s1 = lltype.malloc(S1, flavor='raw') + s2 = lltype.malloc(S1, flavor='raw') + s3 = lltype.malloc(S1, flavor='raw') + s1.x = 111 + s2.x = 222 + s3.x = 333 + s1.p = s2 + s2.p = s3 + s3.p = lltype.nullptr(S1) + sc1 = lltype2ctypes(s1) + sc2 = sc1.contents.p + sc3 = sc2.contents.p + assert not sc3.contents.p + assert sc1.contents.x == 111 + assert sc2.contents.x == 222 + assert sc3.contents.x == 333 + sc3.contents.x += 1 + assert s3.x == 334 + s3.x += 2 + assert sc3.contents.x == 336 + lltype.free(s1, flavor='raw') + lltype.free(s2, flavor='raw') + lltype.free(s3, flavor='raw') + # a self-cycle + s1 = lltype.malloc(S1, flavor='raw') + s1.x = 12 + s1.p = s1 + sc1 = lltype2ctypes(s1) + assert sc1.contents.x == 12 + assert (ctypes.addressof(sc1.contents.p.contents) == + ctypes.addressof(sc1.contents)) + s1.x *= 5 + assert sc1.contents.p.contents.p.contents.p.contents.x == 60 + lltype.free(s1, flavor='raw') + # a longer cycle + s1 = lltype.malloc(S1, flavor='raw') + s2 = lltype.malloc(S1, flavor='raw') + s1.x = 111 + s1.p = s2 + s2.x = 222 + s2.p = s1 + sc1 = lltype2ctypes(s1) + assert sc1.contents.x == 111 + assert sc1.contents.p.contents.x == 222 + assert (ctypes.addressof(sc1.contents.p.contents) != + ctypes.addressof(sc1.contents)) + assert (ctypes.addressof(sc1.contents.p.contents.p.contents) == + ctypes.addressof(sc1.contents)) + lltype.free(s1, flavor='raw') + lltype.free(s2, flavor='raw') + + def test_indirect_recursive_struct(self): + S2Forward = lltype.ForwardReference() + S1 = lltype.Struct('S1', ('p', lltype.Ptr(S2Forward))) + A2 = lltype.Array(lltype.Ptr(S1), hints={'nolength': True}) + S2 = lltype.Struct('S2', ('a', lltype.Ptr(A2))) + S2Forward.become(S2) + s1 = lltype.malloc(S1, flavor='raw') + a2 = lltype.malloc(A2, 10, flavor='raw') + s2 = lltype.malloc(S2, flavor='raw') + s2.a = a2 + a2[5] = s1 + s1.p = s2 + ac2 = lltype2ctypes(a2, normalize=False) + sc1 = ac2.contents.items[5] + sc2 = sc1.contents.p + assert (ctypes.addressof(sc2.contents.a.contents) == + ctypes.addressof(ac2.contents)) + lltype.free(s1, flavor='raw') + lltype.free(a2, flavor='raw') + lltype.free(s2, flavor='raw') From arigo at codespeak.net Tue Sep 18 09:57:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 09:57:02 +0200 (CEST) Subject: [pypy-svn] r46705 - pypy/dist/pypy/rlib Message-ID: <20070918075702.C3BEC80E3@code0.codespeak.net> Author: arigo Date: Tue Sep 18 09:57:02 2007 New Revision: 46705 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py Log: These things were forward pointers. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Tue Sep 18 09:57:02 2007 @@ -186,8 +186,8 @@ CConfig.size_t = platform.SimpleType('size_t', rffi.INT) CConfig.ssize_t = platform.SimpleType('ssize_t', rffi.INT) CConfig.socklen_t = platform.SimpleType('socklen_t', rffi.INT) -sockaddr_ptr = rffi.CStructPtr('sockaddr') -addrinfo_ptr = rffi.CStructPtr('addrinfo') +sockaddr_ptr = lltype.Ptr(lltype.ForwardReference()) +addrinfo_ptr = lltype.Ptr(lltype.ForwardReference()) # struct types CConfig.sockaddr = platform.Struct('struct sockaddr', @@ -288,8 +288,8 @@ pass cConfig.__dict__.update(platform.configure(CConfig)) -sockaddr_ptr = lltype.Ptr(cConfig.sockaddr) -addrinfo_ptr = lltype.Ptr(cConfig.addrinfo) +sockaddr_ptr.TO.become(cConfig.sockaddr) +addrinfo_ptr.TO.become(cConfig.addrinfo) # HACK HACK HACK if _MS_WINDOWS: From arigo at codespeak.net Tue Sep 18 10:04:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 10:04:32 +0200 (CEST) Subject: [pypy-svn] r46707 - pypy/dist/pypy/rlib Message-ID: <20070918080432.E9F5B8133@code0.codespeak.net> Author: arigo Date: Tue Sep 18 10:04:26 2007 New Revision: 46707 Modified: pypy/dist/pypy/rlib/rsocket.py Log: Complete a comment. Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Tue Sep 18 10:04:26 2007 @@ -10,6 +10,9 @@ # - address families other than AF_INET, AF_INET6, AF_UNIX # - methods makefile(), # - SSL +# +# It's unclear if makefile() and SSL support belong here or only as +# app-level code for PyPy. from pypy.rlib.objectmodel import instantiate from pypy.rlib import _rsocket_ctypes as _c From arigo at codespeak.net Tue Sep 18 10:24:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 10:24:47 +0200 (CEST) Subject: [pypy-svn] r46708 - in pypy/dist/pypy/translator/c: . test Message-ID: <20070918082447.A323E80F5@code0.codespeak.net> Author: arigo Date: Tue Sep 18 10:24:47 2007 New Revision: 46708 Modified: pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/test/test_lltyped.py Log: Uh oh. Prebuilt arrays with the 'nolength' hint were no rendered correctly. Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Tue Sep 18 10:24:47 2007 @@ -523,15 +523,19 @@ '%sgcheader%d' % (decoration, i)) for line in lines: yield line + if self.T._hints.get('nolength', False): + length = '' + else: + length = '%d, ' % len(self.obj.items) if self.T.OF is Void or len(self.obj.items) == 0: - yield '\t%d' % len(self.obj.items) + yield '\t%s' % length.rstrip(', ') yield '}' elif self.T.OF == Char: - yield '\t%d, %s' % (len(self.obj.items), - c_char_array_constant(''.join(self.obj.items))) + yield '\t%s%s' % (length, + c_char_array_constant(''.join(self.obj.items))) yield '}' else: - yield '\t%d, {' % len(self.obj.items) + yield '\t%s{' % length for j in range(len(self.obj.items)): value = self.obj.items[j] lines = generic_initializationexpr(self.db, value, Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Tue Sep 18 10:24:47 2007 @@ -340,3 +340,40 @@ fn = self.getcompiled(f, [int]) res = fn(100) assert res == 100 + len(list(names)) + + def test_array_nolength(self): + A = Array(Signed, hints={'nolength': True}) + a1 = malloc(A, 3, immortal=True) + a1[0] = 30 + a1[1] = 300 + a1[2] = 3000 + + def f(n): + a2 = malloc(A, n, flavor='raw') + for i in range(n): + a2[i] = a1[i % 3] + i + res = a2[n // 2] + free(a2, flavor='raw') + return res + + fn = self.getcompiled(f, [int]) + res = fn(100) + assert res == 3050 + + def test_gcarray_nolength(self): + A = GcArray(Signed, hints={'nolength': True}) + a1 = malloc(A, 3, immortal=True) + a1[0] = 30 + a1[1] = 300 + a1[2] = 3000 + + def f(n): + a2 = malloc(A, n) + for i in range(n): + a2[i] = a1[i % 3] + i + res = a2[n // 2] + return res + + fn = self.getcompiled(f, [int]) + res = fn(100) + assert res == 3050 From arigo at codespeak.net Tue Sep 18 10:34:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 10:34:49 +0200 (CEST) Subject: [pypy-svn] r46710 - pypy/dist/pypy/translator/c Message-ID: <20070918083449.7D05A810B@code0.codespeak.net> Author: arigo Date: Tue Sep 18 10:34:48 2007 New Revision: 46710 Modified: pypy/dist/pypy/translator/c/node.py Log: With this, c_char_array_constant() will again generally produce human-greppable string literals. Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Tue Sep 18 10:34:48 2007 @@ -531,8 +531,10 @@ yield '\t%s' % length.rstrip(', ') yield '}' elif self.T.OF == Char: - yield '\t%s%s' % (length, - c_char_array_constant(''.join(self.obj.items))) + s = ''.join(self.obj.items) + if self.T._hints.get('isrpystring', False): + s += '\x00' + yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: yield '\t%s{' % length From arigo at codespeak.net Tue Sep 18 10:55:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 10:55:05 +0200 (CEST) Subject: [pypy-svn] r46713 - pypy/dist/pypy/translator/c Message-ID: <20070918085505.E7D73811E@code0.codespeak.net> Author: arigo Date: Tue Sep 18 10:54:55 2007 New Revision: 46713 Modified: pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py Log: Reduce (a bit) code duplication in genc.py. Allow DefNodes to specify exactly which forward declaration they would like to have, if any. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Tue Sep 18 10:54:55 2007 @@ -444,31 +444,12 @@ name = 'structdef.h' fi = self.makefile(name) print >> f, '#include "%s"' % name - print >> fi, '/***********************************************************/' - print >> fi, '/*** Structure definitions ***/' - print >> fi - for node in structdeflist: - if getattr(node, 'is_external', False): - continue - print >> fi, '%s %s;' % (node.typetag, node.name) - print >> fi - for node in structdeflist: - for line in node.definition(): - print >> fi, line - print >> fi - print >> fi, '/***********************************************************/' + gen_structdef(fi, database) fi.close() name = 'forwarddecl.h' fi = self.makefile(name) print >> f, '#include "%s"' % name - print >> fi, '/***********************************************************/' - print >> fi, '/*** Forward declarations ***/' - print >> fi - for node in database.globalcontainers(): - for line in node.forward_declaration(): - print >> fi, line - print >> fi - print >> fi, '/***********************************************************/' + gen_forwarddecl(fi, database) fi.close() # @@ -552,27 +533,23 @@ print >> f -# this function acts as the fallback for small sources for now. -# Maybe we drop this completely if source splitting is the way -# to go. Currently, I'm quite fine with keeping a working fallback. - -def gen_readable_parts_of_main_c_file(f, database, preimplementationlines=[]): - # - # All declarations - # +def gen_structdef(f, database): structdeflist = database.getstructdeflist() - print >> f print >> f, '/***********************************************************/' print >> f, '/*** Structure definitions ***/' print >> f for node in structdeflist: - if node.name and not getattr(node, 'is_external', False): + if hasattr(node, 'forward_decl'): + if node.forward_decl: + print >> f, node.forward_decl + else: print >> f, '%s %s;' % (node.typetag, node.name) print >> f for node in structdeflist: for line in node.definition(): print >> f, line - print >> f + +def gen_forwarddecl(f, database): print >> f, '/***********************************************************/' print >> f, '/*** Forward declarations ***/' print >> f @@ -580,6 +557,20 @@ for line in node.forward_declaration(): print >> f, line +# this function acts as the fallback for small sources for now. +# Maybe we drop this completely if source splitting is the way +# to go. Currently, I'm quite fine with keeping a working fallback. +# XXX but we need to reduce code duplication. + +def gen_readable_parts_of_main_c_file(f, database, preimplementationlines=[]): + # + # All declarations + # + print >> f + gen_structdef(f, database) + print >> f + gen_forwarddecl(f, database) + # # Implementation of functions and global structures and arrays # Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Tue Sep 18 10:54:55 2007 @@ -36,7 +36,6 @@ class StructDefNode: typetag = 'struct' - is_external = False def __init__(self, db, STRUCT, varlength=1): self.db = db self.STRUCT = STRUCT @@ -56,7 +55,7 @@ self.typetag = '' assert STRUCT._hints.get('external') if self.STRUCT._hints.get('external'): # XXX hack - self.is_external = True + self.forward_decl = None if STRUCT._hints.get('c_name'): self.barename = self.name = STRUCT._hints['c_name'] self.c_struct_field_name = self.verbatim_field_name From arigo at codespeak.net Tue Sep 18 11:58:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 11:58:30 +0200 (CEST) Subject: [pypy-svn] r46716 - in pypy/dist/pypy/translator/c: . src test Message-ID: <20070918095830.C43A88115@code0.codespeak.net> Author: arigo Date: Tue Sep 18 11:58:29 2007 New Revision: 46716 Modified: pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/primitive.py pypy/dist/pypy/translator/c/src/support.h pypy/dist/pypy/translator/c/support.py pypy/dist/pypy/translator/c/test/test_lltyped.py Log: Fix most gcc type warnings by special-casing arrays with no length and no gc headers. Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Tue Sep 18 11:58:29 2007 @@ -9,7 +9,7 @@ from pypy.translator.c.external import CExternalFunctionCodeGenerator from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring from pypy.translator.c.support import cdecl, forward_cdecl, somelettersfrom -from pypy.translator.c.support import c_char_array_constant +from pypy.translator.c.support import c_char_array_constant, barebonearray from pypy.translator.c.primitive import PrimitiveType, isinf, isnan from pypy.translator.c import extfunc @@ -214,15 +214,26 @@ self.name) = db.namespace.uniquename(basename, with_number=with_number, bare=True) self.dependencies = {} - + # a non-gc array with no length doesn't need a 'struct' wrapper at + # all; rffi kind of expects such arrays to be "bare" C arrays. + self.barebone = barebonearray(ARRAY) + if self.barebone: + self.setup() # to get self.itemtypename + typename = self.itemtypename.replace('@', '(@)[%d]' % ( + self.varlength,)) + self.forward_decl = 'typedef %s;' % (cdecl(typename, self.name),) + self.typetag = '' + def setup(self): + if hasattr(self, 'itemtypename'): + return # setup() was already called, likely by __init__ db = self.db ARRAY = self.ARRAY - self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) self.gcinfo # force it to be computed if needs_gcheader(ARRAY): for fname, T in db.gcpolicy.array_gcheader_definition(self): self.gcfields.append((fname, db.gettype(T, who_asks=self))) + self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) def computegcinfo(self): # let the gcpolicy do its own setup @@ -233,7 +244,13 @@ gcinfo = defaultproperty(computegcinfo) def gettype(self): - return 'struct %s @' % self.name + return '%s %s @' % (self.typetag, self.name) + + def getptrtype(self): + if self.barebone: + return self.itemtypename.replace('@', '*@') + else: + return '%s %s *@' % (self.typetag, self.name) def access_expr(self, baseexpr, index): return '%s.items[%d]' % (baseexpr, index) @@ -243,12 +260,16 @@ return self.itemindex_access_expr(baseexpr, index) def itemindex_access_expr(self, baseexpr, indexexpr): - if self.ARRAY._hints.get('nolength', False): + if self.barebone: + return 'RPyBareItem(%s, %s)' % (baseexpr, indexexpr) + elif self.ARRAY._hints.get('nolength', False): return 'RPyNLenItem(%s, %s)' % (baseexpr, indexexpr) else: return 'RPyItem(%s, %s)' % (baseexpr, indexexpr) def definition(self): + if self.barebone: + return gcpolicy = self.db.gcpolicy yield 'struct %s {' % self.name for fname, typename in self.gcfields: @@ -292,7 +313,7 @@ yield 'offsetof(struct %s, length)' % (self.name,) else: yield '-1' - if self.ARRAY.OF is not Void: + if self.ARRAY.OF is not Void and not self.barebone: yield 'offsetof(struct %s, items[0])' % (self.name,) yield 'offsetof(struct %s, items[1])' % (self.name,) else: @@ -504,6 +525,11 @@ if USESLOTS: __slots__ = () + def __init__(self, db, T, obj): + ContainerNode.__init__(self, db, T, obj) + if barebonearray(T): + self.ptrname = self.name + def basename(self): return 'array' @@ -514,6 +540,7 @@ return len(self.obj.items) def initializationexpr(self, decoration=''): + defnode = self.db.gettypedefnode(self.T) yield '{' if needs_gcheader(self.T): for i, thing in enumerate(self.db.gcpolicy.array_gcheader_initdata(self)): @@ -536,7 +563,9 @@ yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: - yield '\t%s{' % length + barebone = barebonearray(self.T) + if not barebone: + yield '\t%s{' % length for j in range(len(self.obj.items)): value = self.obj.items[j] lines = generic_initializationexpr(self.db, value, @@ -544,7 +573,10 @@ '%s%d' % (decoration, j)) for line in lines: yield '\t' + line - yield '} }' + if not barebone: + yield '} }' + else: + yield '}' assert not USESLOTS or '__dict__' not in dir(ArrayNode) Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Tue Sep 18 11:58:29 2007 @@ -7,7 +7,7 @@ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ GCHeaderOffset -from pypy.translator.c.support import cdecl +from pypy.translator.c.support import cdecl, barebonearray # ____________________________________________________________ # @@ -21,13 +21,16 @@ cdecl(db.gettype(value.TYPE), ''), structnode.c_struct_field_name(value.fldname)) elif isinstance(value, ItemOffset): - if value.TYPE != Void: - return '(sizeof(%s) * %s)'%( - cdecl(db.gettype(value.TYPE), ''), value.repeat) + if value.TYPE != Void and value.repeat != 0: + size = 'sizeof(%s)' % (cdecl(db.gettype(value.TYPE), ''),) + if value.repeat != 1: + size = '(%s * %s)' % (size, value.repeat) + return size else: return '0' elif isinstance(value, ArrayItemsOffset): - if isinstance(value.TYPE, FixedSizeArray): + if (isinstance(value.TYPE, FixedSizeArray) or + barebonearray(value.TYPE)): return '0' elif value.TYPE.OF != Void: return 'offsetof(%s, items)'%( Modified: pypy/dist/pypy/translator/c/src/support.h ============================================================================== --- pypy/dist/pypy/translator/c/src/support.h (original) +++ pypy/dist/pypy/translator/c/src/support.h Tue Sep 18 11:58:29 2007 @@ -67,7 +67,7 @@ * it's a "guaranteed" segfault and not one that can be used by * attackers. */ -# define RPyCHECK(x) ((x) || (abort(), 0)) +# define RPyCHECK(x) ((x) || RPyAbort()) # define RPyField(ptr, name) ((RPyCHECK(ptr), (ptr))->name) # define RPyItem(array, index) \ ((RPyCHECK((index) >= 0 && (index) < (array)->length), \ @@ -77,11 +77,24 @@ (ptr))[index]) # define RPyNLenItem(array, index) \ ((RPyCHECK((array) && (index) >= 0), (array))->items[index]) +# define RPyBareItem(array, index) \ + ((RPyCHECK((array) && (index) >= 0), (array)[index]) + +int RPyAbort(void); +#ifndef PYPY_NOT_MAIN_FILE +int RPyAbort(void) { + fprintf(stderr, "Invalid RPython operation (NULL ptr or bad array index)\n"); + abort(); + return 0; +} +#endif + #else # define RPyField(ptr, name) ((ptr)->name) # define RPyItem(array, index) ((array)->items[index]) # define RPyFxItem(ptr, index, fixedsize) ((ptr)[index]) # define RPyNLenItem(array, index) ((array)->items[index]) +# define RPyBareItem(array, index) ((array)[index]) #endif #ifndef PYPY_STANDALONE Modified: pypy/dist/pypy/translator/c/support.py ============================================================================== --- pypy/dist/pypy/translator/c/support.py (original) +++ pypy/dist/pypy/translator/c/support.py Tue Sep 18 11:58:29 2007 @@ -14,6 +14,11 @@ def __init__(self, TYPE): self.TYPE = TYPE +def barebonearray(ARRAY): + """Check if ARRAY is a 'simple' array type, + i.e. doesn't need a length nor GC headers.""" + return ARRAY._hints.get('nolength', False) and ARRAY._gckind != 'gc' + # # helpers Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Tue Sep 18 11:58:29 2007 @@ -347,11 +347,16 @@ a1[0] = 30 a1[1] = 300 a1[2] = 3000 + a1dummy = malloc(A, 2, immortal=True) def f(n): + if n & 1: + src = a1dummy + else: + src = a1 a2 = malloc(A, n, flavor='raw') for i in range(n): - a2[i] = a1[i % 3] + i + a2[i] = src[i % 3] + i res = a2[n // 2] free(a2, flavor='raw') return res @@ -366,11 +371,16 @@ a1[0] = 30 a1[1] = 300 a1[2] = 3000 + a1dummy = malloc(A, 2, immortal=True) def f(n): + if n & 1: + src = a1dummy + else: + src = a1 a2 = malloc(A, n) for i in range(n): - a2[i] = a1[i % 3] + i + a2[i] = src[i % 3] + i res = a2[n // 2] return res From arigo at codespeak.net Tue Sep 18 13:53:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 13:53:52 +0200 (CEST) Subject: [pypy-svn] r46720 - pypy/dist/pypy/translator/c Message-ID: <20070918115352.2B62C813A@code0.codespeak.net> Author: arigo Date: Tue Sep 18 13:53:50 2007 New Revision: 46720 Modified: pypy/dist/pypy/translator/c/primitive.py Log: Special-case the minimum value of r_longlong in the same way as we have to special-case the smallest 'int'. Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Tue Sep 18 13:53:50 2007 @@ -1,6 +1,7 @@ import sys from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic +from pypy.rlib.rarithmetic import r_longlong from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem.llmemory import Address, \ @@ -70,7 +71,11 @@ return '%dULL' % value def name_signedlonglong(value, db): - return '%dLL' % value + maxlonglong = r_longlong.MASK>>1 + if value == -maxlonglong-1: # blame C + return '(-%dLL-1LL)' % maxlonglong + else: + return '%dLL' % value def isinf(x): return x != 0.0 and x / 2 == x From arigo at codespeak.net Tue Sep 18 13:54:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Sep 2007 13:54:02 +0200 (CEST) Subject: [pypy-svn] r46721 - pypy/dist/pypy/translator/c/test Message-ID: <20070918115402.03961813E@code0.codespeak.net> Author: arigo Date: Tue Sep 18 13:54:02 2007 New Revision: 46721 Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py Log: A passing test about arithmetic corner cases. Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Tue Sep 18 13:54:02 2007 @@ -387,3 +387,131 @@ fn = self.getcompiled(f, [int]) res = fn(100) assert res == 3050 + + def test_arithmetic_cornercases(self): + import operator, sys + from pypy.rlib.unroll import unrolling_iterable + from pypy.rlib.rarithmetic import r_longlong, r_ulonglong + + class Undefined: + def __eq__(self, other): + return True + undefined = Undefined() + + def getmin(cls): + if cls is int: + return -sys.maxint-1 + elif cls.SIGNED: + return cls(-(cls.MASK>>1)-1) + else: + return cls(0) + getmin._annspecialcase_ = 'specialize:memo' + + def getmax(cls): + if cls is int: + return sys.maxint + elif cls.SIGNED: + return cls(cls.MASK>>1) + else: + return cls(cls.MASK) + getmax._annspecialcase_ = 'specialize:memo' + maxlonglong = long(getmax(r_longlong)) + + classes = unrolling_iterable([int, r_uint, r_longlong, r_ulonglong]) + operators = unrolling_iterable([operator.add, + operator.sub, + operator.mul, + operator.floordiv, + operator.mod, + operator.lshift, + operator.rshift]) + def f(n): + result = () + for cls in classes: + values = [getmin(cls), getmax(cls)] + for OP in operators: + for x in values: + res1 = OP(x, n) + result += (res1,) + return result + + def assert_eq(a, b): + # for better error messages when it fails + assert len(a) == len(b) + for i in range(len(a)): + assert a[i] == b[i] + + fn = self.getcompiled(f, [int]) + res = fn(1) + print res + assert_eq(res, ( + # int + -sys.maxint, undefined, # add + undefined, sys.maxint-1, # sub + -sys.maxint-1, sys.maxint, # mul + -sys.maxint-1, sys.maxint, # floordiv + 0, 0, # mod + 0, -2, # lshift + (-sys.maxint-1)//2, sys.maxint//2, # rshift + # r_uint + 1, 0, # add + sys.maxint*2+1, sys.maxint*2, # sub + 0, sys.maxint*2+1, # mul + 0, sys.maxint*2+1, # floordiv + 0, 0, # mod + 0, sys.maxint*2, # lshift + 0, sys.maxint, # rshift + # r_longlong + -maxlonglong, undefined, # add + undefined, maxlonglong-1, # sub + -maxlonglong-1, maxlonglong, # mul + -maxlonglong-1, maxlonglong, # floordiv + 0, 0, # mod + 0, -2, # lshift + (-maxlonglong-1)//2, maxlonglong//2, # rshift + # r_ulonglong + 1, 0, # add + maxlonglong*2+1, maxlonglong*2, # sub + 0, maxlonglong*2+1, # mul + 0, maxlonglong*2+1, # floordiv + 0, 0, # mod + 0, maxlonglong*2, # lshift + 0, maxlonglong, # rshift + )) + + res = fn(5) + print res + assert_eq(res, ( + # int + -sys.maxint+4, undefined, # add + undefined, sys.maxint-5, # sub + undefined, undefined, # mul + (-sys.maxint-1)//5, sys.maxint//5, # floordiv + (-sys.maxint-1)%5, sys.maxint%5, # mod + 0, -32, # lshift + (-sys.maxint-1)//32, sys.maxint//32, # rshift + # r_uint + 5, 4, # add + sys.maxint*2-3, sys.maxint*2-4, # sub + 0, sys.maxint*2-3, # mul + 0, (sys.maxint*2+1)//5, # floordiv + 0, (sys.maxint*2+1)%5, # mod + 0, sys.maxint*2-30, # lshift + 0, sys.maxint>>4, # rshift + # r_longlong + -maxlonglong+4, undefined, # add + undefined, maxlonglong-5, # sub + undefined, undefined, # mul + (-maxlonglong-1)//5, maxlonglong//5, # floordiv + (-maxlonglong-1)%5, maxlonglong%5, # mod + 0, -32, # lshift + (-maxlonglong-1)//32, maxlonglong//32,# rshift + # r_ulonglong + 5, 4, # add + maxlonglong*2-3, maxlonglong*2-4, # sub + 0, maxlonglong*2-3, # mul + 0, (maxlonglong*2+1)//5, # floordiv + 0, (maxlonglong*2+1)%5, # mod + 0, maxlonglong*2-30, # lshift + 0, maxlonglong>>4, # rshift + )) From tismer at codespeak.net Tue Sep 18 15:59:47 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Tue, 18 Sep 2007 15:59:47 +0200 (CEST) Subject: [pypy-svn] r46725 - pypy/dist/pypy/rpython/module Message-ID: <20070918135947.978B98131@code0.codespeak.net> Author: tismer Date: Tue Sep 18 15:59:43 2007 New Revision: 46725 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: activating special case for windows which has different names for the stat family - thanks to arigo Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Tue Sep 18 15:59:43 2007 @@ -187,7 +187,7 @@ ARG1 = rffi.CCHARP else: ARG1 = rffi.INT - os_mystat = rffi.llexternal(name, [ARG1, STRUCT_STAT], rffi.INT, + os_mystat = rffi.llexternal(c_func_name, [ARG1, STRUCT_STAT], rffi.INT, includes=INCLUDES) def os_mystat_llimpl(arg): From arigo at codespeak.net Wed Sep 19 13:17:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 13:17:53 +0200 (CEST) Subject: [pypy-svn] r46729 - pypy/dist/pypy/objspace/flow/test Message-ID: <20070919111753.AF927815A@code0.codespeak.net> Author: arigo Date: Wed Sep 19 13:17:52 2007 New Revision: 46729 Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py Log: A very old test that was not actually testing what it was meant to. Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/flow/test/test_objspace.py Wed Sep 19 13:17:52 2007 @@ -75,10 +75,10 @@ def ifthenelse(i, j): if i < 0: i = j - return g(i) + 1 + return user_defined_function(i) + 1 def test_ifthenelse(self): - x = self.codetest(self.simplebranch) + x = self.codetest(self.ifthenelse) #__________________________________________________________ def loop(x): From arigo at codespeak.net Wed Sep 19 16:39:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 16:39:32 +0200 (CEST) Subject: [pypy-svn] r46734 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070919143932.AE2468174@code0.codespeak.net> Author: arigo Date: Wed Sep 19 16:39:31 2007 New Revision: 46734 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: - support for None -> NULL conversion in the fullauto string policy - add a helper rffi.structcopy() to copy a complete structure into another Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Sep 19 16:39:31 2007 @@ -73,7 +73,11 @@ real_args = real_args + (float(args[i]),) elif tp is CCHARP and (stringpolicy == 'fullauto' or stringpolicy == 'autocast'): - ll_str = str2charp(args[i]) + s = args[i] + if s is None: + ll_str = lltype.nullptr(CCHARP.TO) + else: + ll_str = str2charp(s) real_args = real_args + (ll_str,) else: real_args = real_args + (args[i],) @@ -94,7 +98,8 @@ if stringpolicy == 'fullauto': for i, tp in unrolling_arg_tps: if tp is CCHARP: - lltype.free(to_free[i], flavor='raw') + if to_free[i]: + lltype.free(to_free[i], flavor='raw') return result wrapper._always_inline_ = True # for debugging, stick ll func ptr to that @@ -186,6 +191,7 @@ def CArray(tp): return lltype.Array(tp, hints={'nolength': True}) +CArray._annspecialcase_ = 'specialize:memo' def COpaque(name, hints=None, **kwds): if hints is None: @@ -305,10 +311,13 @@ return size, unsigned def sizeof(tp): + # XXX see also llmemory.sizeof() for the symbolic size of structures. + # we could also extend the code below to return the computed size + # of structures as found by rffi_platform. if isinstance(tp, lltype.FixedSizeArray): return sizeof(tp.OF) * tp.length if isinstance(tp, lltype.Ptr): - tp = ULONG + tp = ULONG # XXX! if tp is lltype.Char: return 1 if tp is lltype.Float: @@ -351,3 +360,31 @@ v_name = hop.inputconst(lltype.Void, name) hop.genop('setfield', [v_ptr, v_name, v_arg]) return v_ptr + + +def structcopy(pdst, psrc): + """Copy all the fields of the structure given by 'psrc' + into the structure given by 'pdst'. + """ + copy_fn = _get_structcopy_fn(lltype.typeOf(pdst), lltype.typeOf(psrc)) + copy_fn(pdst, psrc) +structcopy._annspecialcase_ = 'specialize:ll' + +def _get_structcopy_fn(PDST, PSRC): + assert PDST == PSRC + if isinstance(PDST.TO, lltype.Struct): + STRUCT = PDST.TO + fields = [(name, STRUCT._flds[name]) for name in STRUCT._names] + unrollfields = unrolling_iterable(fields) + + def copyfn(pdst, psrc): + for name, TYPE in unrollfields: + if isinstance(TYPE, lltype.ContainerType): + structcopy(getattr(pdst, name), getattr(psrc, name)) + else: + setattr(pdst, name, getattr(psrc, name)) + + return copyfn + else: + raise NotImplementedError('structcopy: type %r' % (PDST.TO,)) +_get_structcopy_fn._annspecialcase_ = 'specialize:memo' Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Wed Sep 19 16:39:31 2007 @@ -313,6 +313,30 @@ assert f() == 3 assert interpret(f, []) == 3 +def test_structcopy(): + X2 = lltype.Struct('X2', ('x', INT)) + X1 = lltype.Struct('X1', ('a', INT), ('x2', X2), ('p', lltype.Ptr(X2))) + def f(): + p2 = make(X2, x=123) + p1 = make(X1, a=5, p=p2) + p1.x2.x = 456 + p1bis = make(X1) + p2bis = make(X2) + structcopy(p1bis, p1) + assert p1bis.a == 5 + assert p1bis.x2.x == 456 + assert p1bis.p == p2 + structcopy(p2bis, p2) + res = p2bis.x + lltype.free(p2bis, flavor='raw') + lltype.free(p1bis, flavor='raw') + lltype.free(p2, flavor='raw') + lltype.free(p1, flavor='raw') + return res + assert f() == 123 + res = interpret(f, []) + assert res == 123 + def test_implicit_cast(): z = llexternal('z', [USHORT, ULONG, USHORT, DOUBLE], USHORT) From arigo at codespeak.net Wed Sep 19 16:45:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 16:45:11 +0200 (CEST) Subject: [pypy-svn] r46735 - in pypy/dist/pypy/rlib: . test Message-ID: <20070919144511.E6B008178@code0.codespeak.net> Author: arigo Date: Wed Sep 19 16:45:11 2007 New Revision: 46735 Added: pypy/dist/pypy/rlib/rsocket_rffi.py - copied, changed from r46707, pypy/dist/pypy/rlib/rsocket.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py - copied, changed from r46700, pypy/dist/pypy/rlib/test/test_rsocket.py Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py Log: Starting a version of rsocket based on _rsocket_rffi. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Wed Sep 19 16:45:11 2007 @@ -376,7 +376,9 @@ fd_set = cConfig.fd_set #c_int_size = sizeof(rffi.INT) -external = rffi.llexternal +def external(*args, **kwds): + kwds.setdefault('stringpolicy', 'fullauto') + return rffi.llexternal(*args, **kwds) if _POSIX: strerror = external('strerror', [rffi.INT], CCHARP) @@ -396,7 +398,8 @@ if not MS_WINDOWS: getaddrinfo = external('getaddrinfo', [CCHARP, CCHARP, - addrinfo_ptr, rffi.CArray(addrinfo_ptr)], rffi.INT) + addrinfo_ptr, + lltype.Ptr(rffi.CArray(addrinfo_ptr))], rffi.INT) freeaddrinfo = external('freeaddrinfo', [addrinfo_ptr], lltype.Void) getnameinfo = external('getnameinfo', [sockaddr_ptr, socklen_t, CCHARP, size_t, CCHARP, size_t, rffi.INT], rffi.INT) From cfbolz at codespeak.net Wed Sep 19 17:04:56 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 19 Sep 2007 17:04:56 +0200 (CEST) Subject: [pypy-svn] r46736 - pypy/dist/pypy/tool/pytest Message-ID: <20070919150456.6FD9B817B@code0.codespeak.net> Author: cfbolz Date: Wed Sep 19 17:04:54 2007 New Revision: 46736 Modified: pypy/dist/pypy/tool/pytest/appsupport.py Log: issue319 resolved make the fake AppCode behave a bit more like py.code.Code to make py.test happy when --tb=short is used. Modified: pypy/dist/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/dist/pypy/tool/pytest/appsupport.py (original) +++ pypy/dist/pypy/tool/pytest/appsupport.py Wed Sep 19 17:04:54 2007 @@ -10,7 +10,8 @@ class AppCode(object): def __init__(self, space, pycode): - self.code = space.unwrap(space.getattr(pycode, space.wrap('co_code'))) + self.code == space.unwrap(space.getattr(pycode, space.wrap('co_code'))) + self.raw = self.code self.w_file = space.getattr(pycode, space.wrap('co_filename')) self.name = space.getattr(pycode, space.wrap('co_name')) self.firstlineno = space.unwrap(space.getattr(pycode, space.wrap('co_firstlineno'))) From arigo at codespeak.net Wed Sep 19 17:27:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 17:27:10 +0200 (CEST) Subject: [pypy-svn] r46737 - in pypy/dist/pypy: rlib rpython/lltypesystem rpython/tool Message-ID: <20070919152710.60A438101@code0.codespeak.net> Author: arigo Date: Wed Sep 19 17:27:10 2007 New Revision: 46737 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Minor progress. I check this in while despairing about the quantity of integer casts that might be needed... Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Wed Sep 19 17:27:10 2007 @@ -402,7 +402,8 @@ lltype.Ptr(rffi.CArray(addrinfo_ptr))], rffi.INT) freeaddrinfo = external('freeaddrinfo', [addrinfo_ptr], lltype.Void) getnameinfo = external('getnameinfo', [sockaddr_ptr, socklen_t, CCHARP, - size_t, CCHARP, size_t, rffi.INT], rffi.INT) + size_t, CCHARP, size_t, rffi.INT], rffi.INT, + stringpolicy='noauto') htonl = external('htonl', [rffi.UINT], rffi.UINT) htons = external('htons', [rffi.USHORT], rffi.USHORT) Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Wed Sep 19 17:27:10 2007 @@ -18,7 +18,7 @@ from pypy.rlib import _rsocket_rffi as _c from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rpython.lltypesystem.llmemory import sizeof +from pypy.rpython.lltypesystem.rffi import sizeof constants = _c.constants @@ -49,6 +49,10 @@ class __metaclass__(type): def __new__(cls, name, bases, dict): family = dict.get('family') + if family is not None: + # lltype is picky about types... + family = rffi.cast(_c.sockaddr.c_sa_family, family) + dict['family'] = family A = type.__new__(cls, name, bases, dict) if family is not None: _FAMILIES[family] = A @@ -69,6 +73,7 @@ def setdata(self, addr, addrlen): # initialize self.addr and self.addrlen. 'addr' can be a different # pointer type than exactly sockaddr_ptr, and we cast it for you. + assert not self.addr self.addr = rffi.cast(_c.sockaddr_ptr, addr) self.addrlen = addrlen setdata._annspecialcase_ = 'specialize:ll' @@ -386,7 +391,7 @@ addr = _c.sockaddr_nl(nl_family = AF_NETLINK) addr.nl_pid = pid addr.nl_groups = groups - self._addr_keepalive_netlink = addr + XXX; self._addr_keepalive_netlink = addr self.addr = cast(pointer(addr), _c.sockaddr_ptr).contents self.addrlen = sizeof(addr) @@ -428,10 +433,13 @@ result = instantiate(familyclass(family)) elif result.family != family: raise RSocketError("address family mismatched") - ... - paddr = result._addr_keepalive0 = copy_buffer(cast(addrptr, POINTER(c_char)), addrlen) - result.addr = cast(paddr, _c.sockaddr_ptr).contents - result.addrlen = addrlen + # copy into a new buffer the address that 'addrptr' points to + addrlen = rffi.cast(lltype.Signed, addrlen) + buf = lltype.malloc(rffi.CCHARP.TO, addrlen, flavor='raw') + src = rffi.cast(rffi.CCHARP, addrptr) + for i in range(addrlen): + buf[i] = src[i] + result.setdata(buf, addrlen) return result def makeipv4addr(s_addr, result=None): @@ -439,19 +447,16 @@ result = instantiate(INETAddress) elif result.family != AF_INET: raise RSocketError("address family mismatched") - sin = _c.sockaddr_in(sin_family = AF_INET) # PLAT sin_len - sin.sin_addr.s_addr = s_addr - paddr = cast(pointer(sin), _c.sockaddr_ptr) - result._addr_keepalive1 = sin - result.addr = paddr.contents - result.addrlen = sizeof(_c.sockaddr_in) + sin = rffi.make(_c.sockaddr_in, c_sin_family = AF_INET) # PLAT sin_len + sin.c_sin_addr.s_addr = s_addr + result.setaddr(sin, sizeof(_c.sockaddr_in)) return result def make_null_address(family): klass = familyclass(family) buf = create_string_buffer(klass.maxlen) result = instantiate(klass) - result._addr_keepalive2 = buf + XXX; result._addr_keepalive2 = buf result.addr = cast(buf, _c.sockaddr_ptr).contents result.addrlen = 0 return result, len(buf) @@ -1001,6 +1006,7 @@ canonname, addr)) info = info.c_ai_next + address_to_fill = None # don't fill the same address repeatedly finally: _c.freeaddrinfo(res) return result @@ -1024,14 +1030,20 @@ return protoent.contents.p_proto def getnameinfo(addr, flags): - host = create_string_buffer(NI_MAXHOST) - serv = create_string_buffer(NI_MAXSERV) - error =_c.getnameinfo(pointer(addr.addr), addr.addrlen, - host, len(host), - serv, len(serv), flags) - if error: - raise GAIError(error) - return host.value, serv.value + host = lltype.malloc(rffi.CCHARP.TO, NI_MAXHOST, flavor='raw') + try: + serv = lltype.malloc(rffi.CCHARP.TO, NI_MAXSERV, flavor='raw') + try: + error =_c.getnameinfo(addr.addr, addr.addrlen, + host, NI_MAXHOST, + serv, NI_MAXSERV, flags) + if error: + raise GAIError(error) + return rffi.charp2str(host), rffi.charp2str(serv) + finally: + lltype.free(serv, flavor='raw') + finally: + lltype.free(host, flavor='raw') if hasattr(_c, 'inet_aton'): def inet_aton(ip): Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Sep 19 17:27:10 2007 @@ -311,11 +311,14 @@ return size, unsigned def sizeof(tp): - # XXX see also llmemory.sizeof() for the symbolic size of structures. - # we could also extend the code below to return the computed size - # of structures as found by rffi_platform. if isinstance(tp, lltype.FixedSizeArray): return sizeof(tp.OF) * tp.length + if isinstance(tp, lltype.Struct): + # the hint is present in structures probed by rffi_platform. + size = tp._hints.get('size') + if size is None: + size = llmemory.sizeof(tp) # a symbolic result in this case + return size if isinstance(tp, lltype.Ptr): tp = ULONG # XXX! if tp is lltype.Char: Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Wed Sep 19 17:27:10 2007 @@ -421,6 +421,7 @@ float_class = [rffi.DOUBLE] def _sizeof(tp): + # XXX don't use this! internal purpose only, not really a sane logic if isinstance(tp, lltype.Struct): return sum([_sizeof(i) for i in tp._flds.values()]) return rffi.sizeof(tp) From cfbolz at codespeak.net Wed Sep 19 17:32:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 19 Sep 2007 17:32:46 +0200 (CEST) Subject: [pypy-svn] r46738 - pypy/dist/pypy/tool/pytest Message-ID: <20070919153246.86AA1816D@code0.codespeak.net> Author: cfbolz Date: Wed Sep 19 17:32:46 2007 New Revision: 46738 Modified: pypy/dist/pypy/tool/pytest/appsupport.py Log: fix nonsense checkin. thanks armin for spotting Modified: pypy/dist/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/dist/pypy/tool/pytest/appsupport.py (original) +++ pypy/dist/pypy/tool/pytest/appsupport.py Wed Sep 19 17:32:46 2007 @@ -10,7 +10,7 @@ class AppCode(object): def __init__(self, space, pycode): - self.code == space.unwrap(space.getattr(pycode, space.wrap('co_code'))) + self.code = space.unwrap(space.getattr(pycode, space.wrap('co_code'))) self.raw = self.code self.w_file = space.getattr(pycode, space.wrap('co_filename')) self.name = space.getattr(pycode, space.wrap('co_name')) From cfbolz at codespeak.net Wed Sep 19 17:56:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 19 Sep 2007 17:56:34 +0200 (CEST) Subject: [pypy-svn] r46739 - in pypy/dist/pypy: annotation annotation/test jit/codegen/test rpython rpython/lltypesystem rpython/test Message-ID: <20070919155634.2C2E6817A@code0.codespeak.net> Author: cfbolz Date: Wed Sep 19 17:56:33 2007 New Revision: 46739 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/test/test_annrpython.py pypy/dist/pypy/jit/codegen/test/operation_tests.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/opimpl.py pypy/dist/pypy/rpython/rfloat.py pypy/dist/pypy/rpython/test/snippet.py pypy/dist/pypy/rpython/test/test_rfloat.py Log: issue311 in-progress remove most support for float ** float. Annotator, rtyper is done, some backends missing Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Wed Sep 19 17:56:33 2007 @@ -434,16 +434,12 @@ truediv = div def pow((flt1, flt2), obj3): - return SomeFloat() - pow.can_only_throw = [ZeroDivisionError, ValueError, OverflowError] + raise NotImplementedError("float power not supported, use math.pow") # repeat these in order to copy the 'can_only_throw' attribute inplace_div = div inplace_truediv = truediv - def inplace_pow((flt1, flt2)): - return SomeFloat() - inplace_pow.can_only_throw = [ZeroDivisionError, ValueError, OverflowError] class __extend__(pairtype(SomeList, SomeList)): Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Wed Sep 19 17:56:33 2007 @@ -2829,6 +2829,18 @@ assert isinstance(s, annmodel.SomeInteger) assert not s.is_constant() + def test_float_pow_unsupported(self): + def f(x, y): + x **= y + return x ** y + a = self.RPythonAnnotator() + s = a.build_types(f, [int, int]) + assert isinstance(s, annmodel.SomeInteger) + a = self.RPythonAnnotator() + py.test.raises(NotImplementedError, a.build_types, f, [float, float]) + + + def g(n): return [0,1,2,n] Modified: pypy/dist/pypy/jit/codegen/test/operation_tests.py ============================================================================== --- pypy/dist/pypy/jit/codegen/test/operation_tests.py (original) +++ pypy/dist/pypy/jit/codegen/test/operation_tests.py Wed Sep 19 17:56:33 2007 @@ -320,13 +320,6 @@ assert fp(40.0, 2.0) == fn(40.0, 2.0), op assert fp(25.125, 1.5) == fn(25.125, 1.5), op - def test_float_pow(self): #harder test for llvm - for op, fn in [('x ** y', lambda x, y: x ** y), #not supported in llvm backend - ]: - fp = self.rgen(fn, [float, float], float) - assert fp(40.0, 2.0) == fn(40.0, 2.0), op - assert fp(25.125, 1.5) == fn(25.125, 1.5), op - def test_float_cast(self): #because of different rettype for op, fn in [('bool(x)', lambda x: bool(x)), ('bool(2.0 - x)', lambda x: bool(x - 2.0)), Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Wed Sep 19 17:56:33 2007 @@ -236,8 +236,8 @@ 'float_ne': LLOp(canfold=True), 'float_gt': LLOp(canfold=True), 'float_ge': LLOp(canfold=True), - 'float_pow': LLOp(canfold=True), # don't implement float_mod, use math.fmod instead + # don't implement float_pow, use math.pow instead 'llong_is_true': LLOp(canfold=True), 'llong_neg': LLOp(canfold=True), Modified: pypy/dist/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/opimpl.py Wed Sep 19 17:56:33 2007 @@ -265,12 +265,6 @@ assert -sys.maxint-1 <= b <= sys.maxint return int(b) -def op_float_pow(b,c): - assert type(b) is float - assert type(c) is float - return math.pow(b,c) - - def op_cast_pointer(RESTYPE, obj): checkptr(obj) return lltype.cast_pointer(RESTYPE, obj) Modified: pypy/dist/pypy/rpython/rfloat.py ============================================================================== --- pypy/dist/pypy/rpython/rfloat.py (original) +++ pypy/dist/pypy/rpython/rfloat.py Wed Sep 19 17:56:33 2007 @@ -53,16 +53,7 @@ # 'floordiv' on floats not supported in RPython - def rtype_pow(_, hop): - s_float3 = hop.args_s[2] - if s_float3.is_constant() and s_float3.const is None: - vlist = hop.inputargs(Float, Float, Void)[:2] - return hop.genop('float_pow', vlist, resulttype=Float) - else: - raise TyperError("cannot handle pow with three float arguments") - - def rtype_inplace_pow(_, hop): - return _rtype_template(hop, 'pow') + # pow on floats not supported in RPython #comparisons: eq is_ ne lt le gt ge Modified: pypy/dist/pypy/rpython/test/snippet.py ============================================================================== --- pypy/dist/pypy/rpython/test/snippet.py (original) +++ pypy/dist/pypy/rpython/test/snippet.py Wed Sep 19 17:56:33 2007 @@ -114,9 +114,6 @@ i += abs(i) i &= 255 - i **= n - i += n**3 - i += -n i += +n i += not n Modified: pypy/dist/pypy/rpython/test/test_rfloat.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rfloat.py (original) +++ pypy/dist/pypy/rpython/test/test_rfloat.py Wed Sep 19 17:56:33 2007 @@ -102,12 +102,6 @@ res = self.interpret(fn, []) assert self.float_eq(res, 42.0) - def test_pow(self): - def fn(x, y): - return x**y - res = self.interpret(fn, [2.0, 3.0]) - assert res == 8.0 - def test_exceptions(self): def fn(x, y, z): try: From cfbolz at codespeak.net Wed Sep 19 17:59:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 19 Sep 2007 17:59:50 +0200 (CEST) Subject: [pypy-svn] r46740 - in pypy/dist/pypy/translator: cli js jvm llvm Message-ID: <20070919155950.75A67817A@code0.codespeak.net> Author: cfbolz Date: Wed Sep 19 17:59:48 2007 New Revision: 46740 Modified: pypy/dist/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/js/opcodes.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/llvm/opwriter.py Log: issue311 testing remove float_pow support from backends Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Wed Sep 19 17:59:48 2007 @@ -164,7 +164,6 @@ 'float_ne': _not('ceq'), 'float_gt': 'cgt', 'float_ge': _not('clt'), - 'float_pow': [PushAllArgs, 'call float64 [mscorlib]System.Math::Pow(float64, float64)'], 'llong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], 'llong_neg': 'neg', Modified: pypy/dist/pypy/translator/js/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/js/opcodes.py (original) +++ pypy/dist/pypy/translator/js/opcodes.py Wed Sep 19 17:59:48 2007 @@ -104,7 +104,6 @@ 'int_invert': [PushAllArgs,_Prefix('~')], 'float_neg': [PushAllArgs,_Prefix('-')], - 'float_pow': [PushAllArgs,_CastFun('Math.pow',2)], 'int_abs': [PushAllArgs,_CastFun('Math.abs',1)], 'float_abs': [PushAllArgs,_CastFun('Math.abs',1)], Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Wed Sep 19 17:59:48 2007 @@ -162,7 +162,6 @@ 'float_sub': jvmgen.DSUB, 'float_mul': jvmgen.DMUL, 'float_truediv': jvmgen.DDIV, - 'float_pow': jvmgen.MATHDPOW, 'float_lt': 'dbl_less_than', 'float_le': 'dbl_less_equals', 'float_eq': 'dbl_equals', Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Wed Sep 19 17:59:48 2007 @@ -180,13 +180,6 @@ # XXX could do something about assertions pass - def int_pow(self, opr): - self._generic_pow(opr, "1") - uint_pow = int_pow - - def float_pow(self, opr): - self._generic_pow(opr, "1.0") - def _generic_neg(self, opr, zerostr): self.codewriter.binaryop("sub", opr.retref, opr.argtypes[0], zerostr, opr.argrefs[0]) From cfbolz at codespeak.net Wed Sep 19 18:00:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 19 Sep 2007 18:00:22 +0200 (CEST) Subject: [pypy-svn] r46741 - pypy/dist/pypy/jit/codegen/llvm/test Message-ID: <20070919160022.079CE80DF@code0.codespeak.net> Author: cfbolz Date: Wed Sep 19 18:00:22 2007 New Revision: 46741 Modified: pypy/dist/pypy/jit/codegen/llvm/test/test_operation.py Log: another one Modified: pypy/dist/pypy/jit/codegen/llvm/test/test_operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llvm/test/test_operation.py (original) +++ pypy/dist/pypy/jit/codegen/llvm/test/test_operation.py Wed Sep 19 18:00:22 2007 @@ -69,6 +69,5 @@ test_float_arithmetic = skip #XXX llvmjit.execute() returns an int :-( test_float_cast = skip #XXX llvmjit.execute() returns an int :-( - test_float_pow = skip test_unichar_array = skip test_char_unichar_fields = skip From cfbolz at codespeak.net Wed Sep 19 18:04:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 19 Sep 2007 18:04:19 +0200 (CEST) Subject: [pypy-svn] r46742 - pypy/dist/pypy/module/__builtin__ Message-ID: <20070919160419.D76818183@code0.codespeak.net> Author: cfbolz Date: Wed Sep 19 18:04:17 2007 New Revision: 46742 Modified: pypy/dist/pypy/module/__builtin__/functional.py Log: add docstring to any and all Modified: pypy/dist/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/functional.py (original) +++ pypy/dist/pypy/module/__builtin__/functional.py Wed Sep 19 18:04:17 2007 @@ -109,6 +109,9 @@ def all(space, w_S): + """all(iterable) -> bool + +Return True if bool(x) is True for all values x in the iterable.""" w_iter = space.iter(w_S) while True: try: @@ -124,6 +127,9 @@ def any(space, w_S): + """any(iterable) -> bool + +Return True if bool(x) is True for any x in the iterable.""" w_iter = space.iter(w_S) while True: try: @@ -138,7 +144,6 @@ any.unwrap_spec = [ObjSpace, W_Root] - class W_XRange(Wrappable): def __init__(self, space, start, len, step): self.space = space From arigo at codespeak.net Wed Sep 19 18:59:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 18:59:43 +0200 (CEST) Subject: [pypy-svn] r46743 - in pypy/dist/pypy: annotation rlib rlib/test rpython rpython/lltypesystem Message-ID: <20070919165943.CE801815C@code0.codespeak.net> Author: arigo Date: Wed Sep 19 18:59:40 2007 New Revision: 46743 Modified: pypy/dist/pypy/annotation/policy.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: * A few tests pass! * Added a new specialization policy: ll_and_arg(n), which uses the ll specialization rules but in addition specializes for the value of the argument n. Modified: pypy/dist/pypy/annotation/policy.py ============================================================================== --- pypy/dist/pypy/annotation/policy.py (original) +++ pypy/dist/pypy/annotation/policy.py Wed Sep 19 18:59:40 2007 @@ -89,6 +89,10 @@ from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy return LowLevelAnnotatorPolicy.default_specialize(*args) + def specialize__ll_and_arg(pol, *args): + from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy + return LowLevelAnnotatorPolicy.specialize__ll_and_arg(*args) + def override__ignore(pol, *args): bk = getbookkeeper() return bk.immutablevalue(None) Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Wed Sep 19 18:59:40 2007 @@ -20,6 +20,10 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.lltypesystem.rffi import sizeof +def castto(example, value): + return rffi.cast(lltype.typeOf(example), value) +castto._annspecialcase_ = 'specialize:ll' + constants = _c.constants locals().update(constants) # Define constants from _c @@ -49,10 +53,6 @@ class __metaclass__(type): def __new__(cls, name, bases, dict): family = dict.get('family') - if family is not None: - # lltype is picky about types... - family = rffi.cast(_c.sockaddr.c_sa_family, family) - dict['family'] = family A = type.__new__(cls, name, bases, dict) if family is not None: _FAMILIES[family] = A @@ -82,7 +82,7 @@ """Convert the address to an app-level object.""" # If we don't know the address family, don't raise an # exception -- return it as a tuple. - family = self.addr.c_sa_family + family = rffi.cast(lltype.Signed, self.addr.c_sa_family) datalen = self.addrlen - llmemory.offsetof(_c.sockaddr, 'c_sa_data') rawdata = ''.join([self.addr.c_sa_data[i] for i in range(datalen)]) return space.newtuple([space.wrap(family), @@ -214,9 +214,10 @@ result = instantiate(INETAddress) # store the malloc'ed data into 'result' as soon as possible # to avoid leaks if an exception occurs inbetween - sin = rffi.make(_c.sockaddr_in, c_sin_family = AF_INET) + sin = rffi.make(_c.sockaddr_in) result.setdata(sin, sizeof(_c.sockaddr_in)) # PLAT sin_len + rffi.setintfield(sin, 'c_sin_family', AF_INET) rffi.structcopy(sin.c_sin_addr, in_addr) return result from_in_addr = staticmethod(from_in_addr) @@ -312,8 +313,9 @@ result = instantiate(INET6Address) # store the malloc'ed data into 'result' as soon as possible # to avoid leaks if an exception occurs inbetween - sin = rffi.make(_c.sockaddr_in6, c_sin6_family = AF_INET) + sin = rffi.make(_c.sockaddr_in6) result.setdata(sin, sizeof(_c.sockaddr_in6)) + rffi.setintfield(sin, 'c_sin6_family', AF_INET) rffi.structcopy(sin.c_sin6_addr, in6_addr) return result from_in6_addr = staticmethod(from_in6_addr) @@ -447,9 +449,10 @@ result = instantiate(INETAddress) elif result.family != AF_INET: raise RSocketError("address family mismatched") - sin = rffi.make(_c.sockaddr_in, c_sin_family = AF_INET) # PLAT sin_len - sin.c_sin_addr.s_addr = s_addr - result.setaddr(sin, sizeof(_c.sockaddr_in)) + sin = rffi.make(_c.sockaddr_in) + result.setdata(sin, sizeof(_c.sockaddr_in)) + rffi.setintfield(sin, 'c_sin_family', AF_INET) # PLAT sin_len + rffi.setintfield(sin.c_sin_addr, 'c_s_addr', s_addr) return result def make_null_address(family): Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Wed Sep 19 18:59:40 2007 @@ -1,5 +1,4 @@ import py, errno, sys -py.test.skip("in-progress") from pypy.rlib import rsocket_rffi as rsocket from pypy.rlib.rsocket_rffi import * @@ -21,12 +20,14 @@ assert res == "" def test_unix_addr(): + py.test.skip("in-progress") if getattr(rsocket, 'AF_UNIX', None) is None: py.test.skip('AF_UNIX not supported.') a = UNIXAddress("/tmp/socketname") assert a.get_path() == "/tmp/socketname" def test_netlink_addr(): + py.test.skip("in-progress") if getattr(rsocket, 'AF_NETLINK', None) is None: py.test.skip('AF_NETLINK not supported.') pid = 1 @@ -36,6 +37,7 @@ assert a.get_groups() == group_mask def test_gethostname(): + py.test.skip("in-progress") s = gethostname() assert isinstance(s, str) @@ -45,6 +47,7 @@ assert a.get_host() == "127.0.0.1" def test_socketpair(): + py.test.skip("in-progress") if sys.platform == "win32": py.test.skip('No socketpair on Windows') s1, s2 = socketpair() @@ -59,6 +62,7 @@ s2.close() def test_simple_tcp(): + py.test.skip("in-progress") import thread sock = RSocket() try_ports = [1023] + range(20000, 30000, 437) @@ -97,6 +101,7 @@ s2.close() def test_simple_udp(): + py.test.skip("in-progress") s1 = RSocket(AF_INET, SOCK_DGRAM) try_ports = [1023] + range(20000, 30000, 437) for port in try_ports: @@ -128,6 +133,7 @@ s2.close() def test_nonblocking(): + py.test.skip("in-progress") sock = RSocket() sock.setblocking(False) try_ports = [1023] + range(20000, 30000, 437) @@ -204,6 +210,7 @@ assert found, lst def test_getaddrinfo_no_reverse_lookup(): + py.test.skip("in-progress") # It seems that getaddrinfo never runs a reverse lookup on Linux. # Python2.3 on Windows returns the hostname. lst = getaddrinfo('134.99.112.214', None, flags=AI_CANONNAME) @@ -216,6 +223,7 @@ assert found, lst def test_connect_ex(): + py.test.skip("in-progress") s = RSocket() err = s.connect_ex(INETAddress('0.0.0.0', 0)) # should not work assert err in (errno.ECONNREFUSED, errno.EADDRNOTAVAIL) @@ -251,6 +259,7 @@ assert reuseptr[0] != 0 def test_dup(): + py.test.skip("in-progress") if sys.platform == "win32": skip("dup does not work on Windows") s = RSocket(AF_INET, SOCK_STREAM) @@ -261,6 +270,7 @@ assert s.getsockname().eq(s2.getsockname()) def test_inet_aton(): + py.test.skip("in-progress") assert inet_aton('1.2.3.4') == '\x01\x02\x03\x04' assert inet_aton('127.0.0.1') == '\x7f\x00\x00\x01' tests = ["127.0.0.256", "127.0.0.255555555555555555", "127.2b.0.0", @@ -281,6 +291,7 @@ HOST = 'localhost' def setup_method(self, method): + py.test.skip("in-progress") self.serv = RSocket(AF_INET, SOCK_STREAM) self.serv.setsockopt_int(SOL_SOCKET, SO_REUSEADDR, 1) self.serv.bind(INETAddress(self.HOST, self.PORT)) Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Wed Sep 19 18:59:40 2007 @@ -38,11 +38,14 @@ def __init__(pol, rtyper=None): pol.rtyper = rtyper - def default_specialize(funcdesc, args_s): + def lowlevelspecialize(funcdesc, args_s, key_for_args): key = [] new_args_s = [] - for s_obj in args_s: - if isinstance(s_obj, annmodel.SomePBC): + for i, s_obj in enumerate(args_s): + if i in key_for_args: + key.append(key_for_args[i]) + new_args_s.append(s_obj) + elif isinstance(s_obj, annmodel.SomePBC): assert s_obj.is_constant(), "ambiguous low-level helper specialization" key.append(KeyComp(s_obj.const)) new_args_s.append(s_obj) @@ -57,6 +60,10 @@ flowgraph = funcdesc.cachedgraph(tuple(key)) args_s[:] = new_args_s return flowgraph + lowlevelspecialize = staticmethod(lowlevelspecialize) + + def default_specialize(funcdesc, args_s): + return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, {}) default_specialize = staticmethod(default_specialize) def override__init_opaque_object(pol, s_opaqueptr, s_value): @@ -97,6 +104,14 @@ specialize__ll = default_specialize + def specialize__ll_and_arg(funcdesc, args_s, *argindices): + keys = {} + for i in argindices: + keys[i] = args_s[i].const + return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, + keys) + specialize__ll_and_arg = staticmethod(specialize__ll_and_arg) + def annotate_lowlevel_helper(annotator, ll_function, args_s, policy=None): if policy is None: policy= LowLevelAnnotatorPolicy() Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Sep 19 18:59:40 2007 @@ -391,3 +391,16 @@ else: raise NotImplementedError('structcopy: type %r' % (PDST.TO,)) _get_structcopy_fn._annspecialcase_ = 'specialize:memo' + + +def setintfield(pdst, fieldname, value): + """Maybe temporary: a helper to set an integer field into a structure, + transparently casting between the various integer types. + """ + STRUCT = lltype.typeOf(pdst).TO + TSRC = lltype.typeOf(value) + TDST = getattr(STRUCT, fieldname) + assert isinstance(TSRC, lltype.Number) + assert isinstance(TDST, lltype.Number) + setattr(pdst, fieldname, cast(TDST, value)) +setintfield._annspecialcase_ = 'specialize:ll_and_arg(1)' From arigo at codespeak.net Wed Sep 19 19:18:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 19:18:11 +0200 (CEST) Subject: [pypy-svn] r46744 - in pypy/dist/pypy/rpython: lltypesystem lltypesystem/test tool Message-ID: <20070919171811.EADE0816C@code0.codespeak.net> Author: arigo Date: Wed Sep 19 19:18:11 2007 New Revision: 46744 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Implement rffi.offsetof(), which tries to return a real number for structures built by rffi_platform. Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Sep 19 19:18:11 2007 @@ -311,12 +311,16 @@ return size, unsigned def sizeof(tp): + """Similar to llmemory.sizeof() but tries hard to return a integer + instead of a symbolic value. + """ if isinstance(tp, lltype.FixedSizeArray): return sizeof(tp.OF) * tp.length if isinstance(tp, lltype.Struct): # the hint is present in structures probed by rffi_platform. size = tp._hints.get('size') if size is None: + from pypy.rpython.lltypesystem import llmemory size = llmemory.sizeof(tp) # a symbolic result in this case return size if isinstance(tp, lltype.Ptr): @@ -329,6 +333,23 @@ if tp is lltype.Signed: return ULONG._type.BITS/8 return tp._type.BITS/8 +_annspecialcase_ = 'specialize:memo' + +def offsetof(STRUCT, fieldname): + """Similar to llmemory.offsetof() but tries hard to return a integer + instead of a symbolic value. + """ + # the hint is present in structures probed by rffi_platform. + fieldoffsets = STRUCT._hints.get('fieldoffsets') + if fieldoffsets is not None: + # a numeric result when known + for index, name in enumerate(STRUCT._names): + if name == fieldname: + return fieldoffsets[index] + # a symbolic result as a fallback + from pypy.rpython.lltypesystem import llmemory + return llmemory.offsetof(STRUCT, fieldname) +_annspecialcase_ = 'specialize:memo' # ********************** some helpers ******************* Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Wed Sep 19 19:18:11 2007 @@ -269,6 +269,23 @@ assert not size_and_sign(lltype.Char)[1] assert size_and_sign(UINT)[1] +def test_rffi_offsetof(): + import struct + from pypy.rpython.tool import rffi_platform + S = rffi_platform.getstruct("struct S", + """ + struct S { + short a; + int b, c; + }; """, + [("a", INT), + ("b", INT), + ("c", INT)]) + assert sizeof(S) == struct.calcsize("hii") + assert offsetof(S, "c_a") == 0 + assert offsetof(S, "c_b") == struct.calcsize("hi") - struct.calcsize("i") + assert offsetof(S, "c_c") == struct.calcsize("hii") - struct.calcsize("i") + def test_prebuild_constant(): py.test.skip("WIP") h_source = py.code.Source(""" Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Wed Sep 19 19:18:11 2007 @@ -240,17 +240,20 @@ # build the lltype Structure seen = {} fields = [] - for cell in layout: + fieldoffsets = [] + for offset, cell in enumerate(layout): if cell in seen: continue fields.append((cell.name, cell.ctype)) + fieldoffsets.append(offset) seen[cell] = True name = self.name if name.startswith('struct '): name = name[7:] kwds = {'hints':{'align':info['align'], - 'size':info['size']}} + 'size':info['size'], + 'fieldoffsets':tuple(fieldoffsets)}} return rffi.CStruct(name, *fields, **kwds) class SimpleType(CConfigEntry): From arigo at codespeak.net Wed Sep 19 21:20:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 21:20:27 +0200 (CEST) Subject: [pypy-svn] r46746 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20070919192027.465438093@code0.codespeak.net> Author: arigo Date: Wed Sep 19 21:20:25 2007 New Revision: 46746 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Log: Stop using the deprecated ctypes.SetPointerType(). The new way to make recursive structures in ctypes is to set the _fields_ attribute later. This is a bit delicate but overall it looks like a clean-up. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Wed Sep 19 21:20:25 2007 @@ -21,8 +21,6 @@ _ctypes_cache = {} -_delayed_ptrs = [] -_gettype_recursion = 0 def _setup_ctypes_cache(): from pypy.rpython.lltypesystem import rffi @@ -44,18 +42,22 @@ rffi.SIZE_T: ctypes.c_size_t, }) -def build_ctypes_struct(S, max_n=None): - fields = [] - for fieldname in S._names: - FIELDTYPE = S._flds[fieldname] - if max_n is not None and fieldname == S._arrayfld: - cls = build_ctypes_array(FIELDTYPE, max_n) - else: - cls = get_ctypes_type(FIELDTYPE) - fields.append((fieldname, cls)) +def build_ctypes_struct(S, delayed_builders, max_n=None): + def builder(): + # called a bit later to fill in _fields_ + # (to handle recursive structure pointers) + fields = [] + for fieldname in S._names: + FIELDTYPE = S._flds[fieldname] + if max_n is not None and fieldname == S._arrayfld: + cls = build_ctypes_array(FIELDTYPE, None, max_n) + else: + cls = get_ctypes_type(FIELDTYPE) + fields.append((fieldname, cls)) + CStruct._fields_ = fields class CStruct(ctypes.Structure): - _fields_ = fields + # no _fields_: filled later by builder() def _malloc(cls, n=None): if S._arrayfld is None: @@ -66,7 +68,7 @@ else: if n is None: raise TypeError("%r is variable-sized" % (S,)) - biggercls = build_ctypes_struct(S, n) + biggercls = build_ctypes_struct(S, None, n) bigstruct = biggercls() array = getattr(bigstruct, S._arrayfld) if hasattr(array, 'length'): @@ -77,12 +79,15 @@ CStruct.__name__ = 'ctypes_%s' % (S,) if max_n is not None: CStruct._normalized_ctype = get_ctypes_type(S) + builder() # no need to be lazy here + else: + delayed_builders.append(builder) return CStruct -def build_ctypes_array(A, max_n=0): +def build_ctypes_array(A, delayed_builders, max_n=0): assert max_n >= 0 ITEM = A.OF - ctypes_item = get_ctypes_type(ITEM) + ctypes_item = get_ctypes_type(ITEM, delayed_builders) class CArray(ctypes.Structure): if not A._hints.get('nolength'): @@ -94,7 +99,7 @@ def _malloc(cls, n=None): if not isinstance(n, int): raise TypeError, "array length must be an int" - biggercls = build_ctypes_array(A, n) + biggercls = build_ctypes_array(A, None, n) bigarray = biggercls() if hasattr(bigarray, 'length'): bigarray.length = n @@ -127,27 +132,24 @@ CArray._normalized_ctype = get_ctypes_type(A) return CArray -def get_ctypes_type(T): +def get_ctypes_type(T, delayed_builders=None): try: return _ctypes_cache[T] except KeyError: - global _gettype_recursion - _gettype_recursion += 1 - try: - cls = build_new_ctypes_type(T) - if T not in _ctypes_cache: - _ctypes_cache[T] = cls - else: - # check for buggy recursive structure logic - assert _ctypes_cache[T] is cls - - if _gettype_recursion == 1: - complete_pointer_types() - finally: - _gettype_recursion -= 1 + toplevel = delayed_builders is None + if toplevel: + delayed_builders = [] + cls = build_new_ctypes_type(T, delayed_builders) + if T not in _ctypes_cache: + _ctypes_cache[T] = cls + else: + # check for buggy recursive structure logic + assert _ctypes_cache[T] is cls + if toplevel: + complete_builders(delayed_builders) return cls -def build_new_ctypes_type(T): +def build_new_ctypes_type(T, delayed_builders): if isinstance(T, lltype.Ptr): if isinstance(T.TO, lltype.FuncType): argtypes = [get_ctypes_type(ARG) for ARG in T.TO.ARGS] @@ -156,18 +158,12 @@ else: restype = get_ctypes_type(T.TO.RESULT) return ctypes.CFUNCTYPE(restype, *argtypes) - elif isinstance(T.TO, lltype.Struct): - # for recursive structures: build a forward pointer first - uniquename = 'ctypes_%s_%d' % (T.TO.__name__, len(_ctypes_cache)) - pcls = ctypes.POINTER(uniquename) - _delayed_ptrs.append((pcls, T.TO)) - return pcls else: - return ctypes.POINTER(get_ctypes_type(T.TO)) + return ctypes.POINTER(get_ctypes_type(T.TO, delayed_builders)) elif isinstance(T, lltype.Struct): - return build_ctypes_struct(T) + return build_ctypes_struct(T, delayed_builders) elif isinstance(T, lltype.Array): - return build_ctypes_array(T) + return build_ctypes_array(T, delayed_builders) elif isinstance(T, lltype.OpaqueType): if T.hints.get('external', None) != 'C': raise TypeError("%s is not external" % T) @@ -178,10 +174,9 @@ return _ctypes_cache[T] raise NotImplementedError(T) -def complete_pointer_types(): - while _delayed_ptrs: - pcls, S = _delayed_ptrs.pop() - ctypes.SetPointerType(pcls, get_ctypes_type(S)) +def complete_builders(delayed_builders): + while delayed_builders: + delayed_builders.pop()() def convert_struct(container, cstruct=None): From arigo at codespeak.net Wed Sep 19 21:37:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 21:37:23 +0200 (CEST) Subject: [pypy-svn] r46747 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20070919193723.106EF8174@code0.codespeak.net> Author: arigo Date: Wed Sep 19 21:37:22 2007 New Revision: 46747 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Substructures don't completely work in ll2ctypes - test... Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Wed Sep 19 21:37:22 2007 @@ -425,6 +425,20 @@ sc.contents.s1a.x += 3 assert s1ac.contents.x == 59 assert s.s1a.x == 59 + + py.test.skip("in-progress") + t = ctypes2lltype(lltype.Ptr(BIG), sc) + assert t == s + assert t.s1a == s.s1a + assert t.s1a.x == 59 + s.s1b.x = 8888 + assert t.s1b == s.s1b + assert t.s1b.x == 8888 + t1 = ctypes2lltype(lltype.Ptr(S1), s1ac) + assert t.s1a == t1 + assert t1.x == 59 + t1.x += 1 + assert sc.contents.s1a.x == 60 lltype.free(s, flavor='raw') def test_recursive_struct(self): From arigo at codespeak.net Wed Sep 19 21:37:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 21:37:53 +0200 (CEST) Subject: [pypy-svn] r46748 - pypy/dist/pypy/rlib Message-ID: <20070919193753.5B3BD8182@code0.codespeak.net> Author: arigo Date: Wed Sep 19 21:37:53 2007 New Revision: 46748 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket_rffi.py Log: Some progress. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Wed Sep 19 21:37:53 2007 @@ -211,7 +211,7 @@ CConfig.sockaddr_un = platform.Struct('struct sockaddr_un', [('sun_family', rffi.INT), - ('sun_path', rffi.CFixedArray(rffi.UCHAR, 1))], + ('sun_path', rffi.CFixedArray(rffi.CHAR, 1))], ifdef='AF_UNIX') CConfig.sockaddr_nl = platform.Struct('struct sockaddr_nl', Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Wed Sep 19 21:37:53 2007 @@ -18,7 +18,7 @@ from pypy.rlib import _rsocket_rffi as _c from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rpython.lltypesystem.rffi import sizeof +from pypy.rpython.lltypesystem.rffi import sizeof, offsetof def castto(example, value): return rffi.cast(lltype.typeOf(example), value) @@ -83,7 +83,7 @@ # If we don't know the address family, don't raise an # exception -- return it as a tuple. family = rffi.cast(lltype.Signed, self.addr.c_sa_family) - datalen = self.addrlen - llmemory.offsetof(_c.sockaddr, 'c_sa_data') + datalen = self.addrlen - offsetof(_c.sockaddr, 'c_sa_data') rawdata = ''.join([self.addr.c_sa_data[i] for i in range(datalen)]) return space.newtuple([space.wrap(family), space.wrap(rawdata)]) @@ -333,26 +333,26 @@ maxlen = sizeof(struct) def __init__(self, path): - sun = _c.sockaddr_un(sun_family = AF_UNIX) + sun = rffi.make(_c.sockaddr_un) + rffi.setintfield(sun, 'c_sun_family', AF_UNIX) if _c.linux and path.startswith('\x00'): # Linux abstract namespace extension - if len(path) > sizeof(sun.sun_path): + if len(path) > sizeof(_c.sockaddr_un.c_sun_path): raise RSocketError("AF_UNIX path too long") else: # regular NULL-terminated string - if len(path) >= sizeof(sun.sun_path): + if len(path) >= sizeof(_c.sockaddr_un.c_sun_path): raise RSocketError("AF_UNIX path too long") - sun.sun_path[len(path)] = 0 + sun.c_sun_path[len(path)] = '\x00' for i in range(len(path)): - sun.sun_path[i] = ord(path[i]) - self.sun = sun - self.addr = cast(pointer(sun), _c.sockaddr_ptr).contents - self.addrlen = offsetof(_c.sockaddr_un, 'sun_path') + len(path) + sun.c_sun_path[i] = path[i] + baseofs = offsetof(_c.sockaddr_un, 'c_sun_path') + self.setdata(sun, baseofs + len(path)) def as_sockaddr_un(self): - if self.addrlen <= offsetof(_c.sockaddr_un, 'sun_path'): + if self.addrlen <= offsetof(_c.sockaddr_un, 'c_sun_path'): raise RSocketError("invalid address") - return cast(pointer(self.addr), POINTER(_c.sockaddr_un)).contents + return rffi.cast(lltype.Ptr(_c.sockaddr_un), self.addr) def __repr__(self): try: @@ -362,15 +362,16 @@ def get_path(self): a = self.as_sockaddr_un() - if _c.linux and a.sun_path[0] == 0: + maxlength = self.addrlen - offsetof(_c.sockaddr_un, 'c_sun_path') + if _c.linux and a.c_sun_path[0] == '\x00': # Linux abstract namespace - buf = copy_buffer(cast(pointer(a.sun_path), POINTER(c_char)), - self.addrlen - offsetof(_c.sockaddr_un, - 'sun_path')) - return buf.raw + length = maxlength else: # regular NULL-terminated string - return cast(pointer(a.sun_path), c_char_p).value + length = 0 + while length < maxlength and a.c_sun_path[length] != '\x00': + length += 1 + return ''.join([a.c_sun_path[i] for i in range(length)]) def eq(self, other): # __eq__() is not called by RPython :-/ return (isinstance(other, UNIXAddress) and From arigo at codespeak.net Wed Sep 19 22:22:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 22:22:31 +0200 (CEST) Subject: [pypy-svn] r46749 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070919202231.0FD488173@code0.codespeak.net> Author: arigo Date: Wed Sep 19 22:22:29 2007 New Revision: 46749 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Fix substructures. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Wed Sep 19 22:22:29 2007 @@ -228,6 +228,17 @@ for i in range(container.getlength()): container.items[i] = None +def struct_use_ctypes_storage(container, ctypes_storage): + STRUCT = container._TYPE + assert isinstance(STRUCT, lltype.Struct) + add_storage(container, _struct_mixin, ctypes_storage) + remove_regular_struct_content(container) + for field_name in STRUCT._names: + FIELDTYPE = getattr(STRUCT, field_name) + if isinstance(FIELDTYPE, lltype.ContainerType): + struct_use_ctypes_storage(getattr(container, field_name), + getattr(ctypes_storage, field_name)) + # ____________________________________________________________ # Ctypes-aware subclasses of the _parentable classes @@ -394,10 +405,10 @@ if not cobj: # NULL pointer return lltype.nullptr(T.TO) if isinstance(T.TO, lltype.Struct): - # XXX var-sized structs + if T.TO._arrayfld is not None: + raise NotImplementedError("XXX var-sized structs") container = lltype._struct(T.TO) - add_storage(container, _struct_mixin, cobj.contents) - remove_regular_struct_content(container) + struct_use_ctypes_storage(container, cobj.contents) elif isinstance(T.TO, lltype.Array): if T.TO._hints.get('nolength', False): container = _array_of_unknown_length(T.TO) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Wed Sep 19 22:22:29 2007 @@ -426,7 +426,6 @@ assert s1ac.contents.x == 59 assert s.s1a.x == 59 - py.test.skip("in-progress") t = ctypes2lltype(lltype.Ptr(BIG), sc) assert t == s assert t.s1a == s.s1a From arigo at codespeak.net Wed Sep 19 22:22:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 22:22:57 +0200 (CEST) Subject: [pypy-svn] r46750 - pypy/dist/pypy/rlib/test Message-ID: <20070919202257.74B3A8174@code0.codespeak.net> Author: arigo Date: Wed Sep 19 22:22:57 2007 New Revision: 46750 Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: This test passes now. Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Wed Sep 19 22:22:57 2007 @@ -20,7 +20,6 @@ assert res == "" def test_unix_addr(): - py.test.skip("in-progress") if getattr(rsocket, 'AF_UNIX', None) is None: py.test.skip('AF_UNIX not supported.') a = UNIXAddress("/tmp/socketname") From arigo at codespeak.net Wed Sep 19 22:31:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 22:31:30 +0200 (CEST) Subject: [pypy-svn] r46751 - in pypy/dist/pypy/rlib: . test Message-ID: <20070919203130.86C768184@code0.codespeak.net> Author: arigo Date: Wed Sep 19 22:31:30 2007 New Revision: 46751 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: fixed NETLINKAddress. Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Wed Sep 19 22:31:30 2007 @@ -334,6 +334,8 @@ def __init__(self, path): sun = rffi.make(_c.sockaddr_un) + baseofs = offsetof(_c.sockaddr_un, 'c_sun_path') + self.setdata(sun, baseofs + len(path)) rffi.setintfield(sun, 'c_sun_family', AF_UNIX) if _c.linux and path.startswith('\x00'): # Linux abstract namespace extension @@ -346,8 +348,6 @@ sun.c_sun_path[len(path)] = '\x00' for i in range(len(path)): sun.c_sun_path[i] = path[i] - baseofs = offsetof(_c.sockaddr_un, 'c_sun_path') - self.setdata(sun, baseofs + len(path)) def as_sockaddr_un(self): if self.addrlen <= offsetof(_c.sockaddr_un, 'c_sun_path'): @@ -391,23 +391,22 @@ maxlen = sizeof(struct) def __init__(self, pid, groups): - addr = _c.sockaddr_nl(nl_family = AF_NETLINK) - addr.nl_pid = pid - addr.nl_groups = groups - XXX; self._addr_keepalive_netlink = addr - self.addr = cast(pointer(addr), _c.sockaddr_ptr).contents - self.addrlen = sizeof(addr) + addr = rffi.make(_c.sockaddr_nl) + self.setdata(addr, NETLINKAddress.maxlen) + rffi.setintfield(addr, 'c_nl_family', AF_NETLINK) + rffi.setintfield(addr, 'c_nl_pid', pid) + rffi.setintfield(addr, 'c_nl_groups', groups) def as_sockaddr_nl(self): if self.addrlen != NETLINKAddress.maxlen: raise RSocketError("invalid address") - return cast(pointer(self.addr), POINTER(_c.sockaddr_nl)).contents + return rffi.cast(lltype.Ptr(_c.sockaddr_nl), self.addr) def get_pid(self): - return self.as_sockaddr_nl().nl_pid + return self.as_sockaddr_nl().c_nl_pid def get_groups(self): - return self.as_sockaddr_nl().nl_groups + return self.as_sockaddr_nl().c_nl_groups def __repr__(self): return '' % (self.get_pid(), self.get_groups()) Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Wed Sep 19 22:31:30 2007 @@ -26,7 +26,6 @@ assert a.get_path() == "/tmp/socketname" def test_netlink_addr(): - py.test.skip("in-progress") if getattr(rsocket, 'AF_NETLINK', None) is None: py.test.skip('AF_NETLINK not supported.') pid = 1 From arigo at codespeak.net Wed Sep 19 22:36:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Sep 2007 22:36:44 +0200 (CEST) Subject: [pypy-svn] r46752 - in pypy/dist/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20070919203644.7E8E4818A@code0.codespeak.net> Author: arigo Date: Wed Sep 19 22:36:43 2007 New Revision: 46752 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: * rsocket_rffi.gethostname() * rffi.charp2strn(p, maxlen) Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Wed Sep 19 22:36:43 2007 @@ -446,7 +446,8 @@ sendto = external('sendto', [socketfd_type, rffi.VOIDP, size_t, rffi.INT, sockaddr_ptr, socklen_t], ssize_t) shutdown = external('shutdown', [socketfd_type, rffi.INT], rffi.INT) -gethostname = external('gethostname', [rffi.CCHARP, rffi.INT], rffi.INT) +gethostname = external('gethostname', [rffi.CCHARP, rffi.INT], rffi.INT, + stringpolicy='noauto') gethostbyname = external('gethostbyname', [rffi.CCHARP], lltype.Ptr(cConfig.hostent)) gethostbyaddr = external('gethostbyaddr', [rffi.VOIDP, rffi.INT, rffi.INT], lltype.Ptr(cConfig.hostent)) Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Wed Sep 19 22:36:43 2007 @@ -914,12 +914,15 @@ return defaults.timeout def gethostname(): - buf = create_string_buffer(1024) - res = _c.gethostname(buf, sizeof(buf)-1) - if res < 0: - raise last_error() - buf[sizeof(buf)-1] = '\x00' - return buf.value + size = 1024 + buf = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw') + try: + res = _c.gethostname(buf, size) + if res < 0: + raise last_error() + return rffi.charp2strn(buf, size) + finally: + lltype.free(buf, flavor='raw') def gethostbyname(name): # this is explicitly not working with IPv6, because the docs say it Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Wed Sep 19 22:36:43 2007 @@ -35,7 +35,6 @@ assert a.get_groups() == group_mask def test_gethostname(): - py.test.skip("in-progress") s = gethostname() assert isinstance(s, str) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Sep 19 22:36:43 2007 @@ -274,6 +274,15 @@ i += 1 return "".join(l) +# char* -> str, with an upper bound on the length in case there is no \x00 +def charp2strn(cp, maxlen): + l = [] + i = 0 + while i < maxlen and cp[i] != '\x00': + l.append(cp[i]) + i += 1 + return "".join(l) + # char** CCHARPP = lltype.Ptr(lltype.Array(CCHARP, hints={'nolength': True})) From cfbolz at codespeak.net Thu Sep 20 10:16:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 20 Sep 2007 10:16:46 +0200 (CEST) Subject: [pypy-svn] r46753 - pypy/dist/pypy/interpreter Message-ID: <20070920081646.6958F816C@code0.codespeak.net> Author: cfbolz Date: Thu Sep 20 10:16:44 2007 New Revision: 46753 Modified: pypy/dist/pypy/interpreter/pyopcode.py Log: issue302 testing turn the indirect call in COMPARE_OP into a switch statement. Tests pass, translation works. Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Thu Sep 20 10:16:44 2007 @@ -19,6 +19,7 @@ from pypy.tool.stdlib_opcode import unrolling_opcode_descs from pypy.tool.stdlib_opcode import opcode_method_names from pypy.rlib import rstack # for resume points +from pypy.rlib.unroll import unrolling_iterable def unaryoperation(operationname): """NOT_RPYTHON""" @@ -43,6 +44,23 @@ return func_with_new_name(opimpl, "opcode_impl_for_%s" % operationname) +compare_dispatch_table = [ + "cmp_lt", # "<" + "cmp_le", # "<=" + "cmp_eq", # "==" + "cmp_ne", # "!=" + "cmp_gt", # ">" + "cmp_ge", # ">=" + "cmp_in", + "cmp_not_in", + "cmp_is", + "cmp_is_not", + "cmp_exc_match", + ] + +unrolling_compare_dispatch_table = unrolling_iterable( + enumerate(compare_dispatch_table)) + class __extend__(pyframe.PyFrame): """A PyFrame that knows about interpretation of standard Python opcodes @@ -679,28 +697,16 @@ def cmp_exc_match(f, w_1, w_2): return f.space.newbool(f.space.exception_match(w_1, w_2)) - compare_dispatch_table = [ - cmp_lt, # "<" - cmp_le, # "<=" - cmp_eq, # "==" - cmp_ne, # "!=" - cmp_gt, # ">" - cmp_ge, # ">=" - cmp_in, - cmp_not_in, - cmp_is, - cmp_is_not, - cmp_exc_match, - ] def COMPARE_OP(f, testnum, *ignored): w_2 = f.popvalue() w_1 = f.popvalue() - table = hint(f.compare_dispatch_table, deepfreeze=True) - try: - testfn = table[testnum] - except IndexError: + w_result = None + for i, attr in unrolling_compare_dispatch_table: + if i == testnum: + w_result = getattr(f, attr)(w_1, w_2) + break + else: raise BytecodeCorruption, "bad COMPARE_OP oparg" - w_result = testfn(f, w_1, w_2) f.pushvalue(w_result) def IMPORT_NAME(f, nameindex, *ignored): From cfbolz at codespeak.net Thu Sep 20 11:44:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 20 Sep 2007 11:44:35 +0200 (CEST) Subject: [pypy-svn] r46754 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20070920094435.52D1A816E@code0.codespeak.net> Author: cfbolz Date: Thu Sep 20 11:44:34 2007 New Revision: 46754 Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py Log: add a test to show that graphanalyze does not handle oosend correctly Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_canraise.py Thu Sep 20 11:44:34 2007 @@ -6,6 +6,9 @@ class BaseTestCanRaise(object): type_system = None + def _skip_oo(self, reason): + if self.type_system == 'ootype': + py.test.skip("ootypesystem doesn't support %s, yet" % reason) def translate(self, func, sig): t = TranslationContext() @@ -70,6 +73,28 @@ result = ra.can_raise(hgraph.startblock.operations[0]) assert result + def test_method(self): + self._skip_oo("oosend analysis") + class A(object): + def f(x): + return 1 + class B(A): + def f(x): + return 2 + def f(a): + return a.f() + def h(x): + if x: + a = A() + else: + a = B() + return f(a) + t, ra = self.translate(h, [int]) + hgraph = graphof(t, h) + # fiiiish :-( + result = ra.can_raise(hgraph.startblock.exits[0].target.exits[0].target.operations[0]) + assert not result + def test_instantiate(self): # instantiate is interesting, because it leads to one of the few cases of # an indirect call without a list of graphs From arigo at codespeak.net Thu Sep 20 12:22:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 12:22:06 +0200 (CEST) Subject: [pypy-svn] r46755 - in pypy/dist/pypy: annotation rpython rpython/test Message-ID: <20070920102206.4AA7B8198@code0.codespeak.net> Author: arigo Date: Thu Sep 20 12:22:04 2007 New Revision: 46755 Modified: pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/test/test_llann.py Log: Allow *args in low-level helpers, by making it compatible with specialize:ll. Based on a small refactoring which should allow other specializations to be easily combined with *args. I think I ran the relevant tests but I'll take the blame if not. Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Thu Sep 20 12:22:04 2007 @@ -8,7 +8,7 @@ from pypy.objspace.flow.model import Constant, checkgraph from pypy.annotation import model as annmodel -def default_specialize(funcdesc, args_s): +def flatten_star_args(funcdesc, args_s): argnames, vararg, kwarg = funcdesc.signature assert not kwarg, "functions with ** arguments are not supported" if vararg: @@ -49,26 +49,29 @@ checkgraph(graph) return graph - key, name_suffix = access_direct_key(nb_extra_args, flattened_s) - return funcdesc.cachedgraph(key, - alt_name='%s_star%d%s' % (funcdesc.name, - nb_extra_args, - name_suffix), - builder=builder) + key = nb_extra_args + name_suffix = '_star%d' % (nb_extra_args,) + return flattened_s, key, name_suffix, builder + else: - key, name_suffix = access_direct_key(None, args_s) - if name_suffix: - alt_name = '%s%s' % (funcdesc.name, name_suffix) - else: - alt_name = None - return funcdesc.cachedgraph(key, alt_name=alt_name) + return args_s, None, '', None -def access_direct_key(key, args_s): +def default_specialize(funcdesc, args_s): + # first flatten the *args + args_s, key, name_suffix, builder = flatten_star_args(funcdesc, args_s) + # two versions: a regular one and one for instances with 'access_directly' for s_obj in args_s: if (isinstance(s_obj, annmodel.SomeInstance) and 'access_directly' in s_obj.flags): - return (AccessDirect, key), '_AccessDirect' - return key, '' + key = (AccessDirect, key) + name_suffix += '_AccessDirect' + break + # done + if name_suffix: + alt_name = '%s%s' % (funcdesc.name, name_suffix) + else: + alt_name = None + return funcdesc.cachedgraph(key, alt_name=alt_name, builder=builder) class AccessDirect(object): """marker for specialization: set when any arguments is a SomeInstance Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Thu Sep 20 12:22:04 2007 @@ -6,6 +6,7 @@ from pypy.tool.sourcetools import valid_identifier from pypy.annotation import model as annmodel from pypy.annotation.policy import AnnotatorPolicy, Sig +from pypy.annotation.specialize import flatten_star_args from pypy.rpython.lltypesystem import lltype from pypy.rpython import extfunctable, extregistry from pypy.objspace.flow.model import Constant @@ -39,7 +40,8 @@ pol.rtyper = rtyper def lowlevelspecialize(funcdesc, args_s, key_for_args): - key = [] + args_s, key, ignored, builder = flatten_star_args(funcdesc, args_s) + key = [key] new_args_s = [] for i, s_obj in enumerate(args_s): if i in key_for_args: @@ -57,7 +59,7 @@ # passing non-low-level types to a ll_* function is allowed # for module/ll_* key.append(s_obj.__class__) - flowgraph = funcdesc.cachedgraph(tuple(key)) + flowgraph = funcdesc.cachedgraph(tuple(key), builder=builder) args_s[:] = new_args_s return flowgraph lowlevelspecialize = staticmethod(lowlevelspecialize) Modified: pypy/dist/pypy/rpython/test/test_llann.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llann.py (original) +++ pypy/dist/pypy/rpython/test/test_llann.py Thu Sep 20 12:22:04 2007 @@ -10,6 +10,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.test.test_llinterp import interpret from pypy.objspace.flow import FlowObjSpace +from pypy.conftest import option # helpers @@ -34,6 +35,8 @@ def annotate(self, ll_function, argtypes): self.a = self.RPythonAnnotator() graph = annotate_lowlevel_helper(self.a, ll_function, argtypes) + if option.view: + self.a.translator.view() return self.a.binding(graph.getreturnvar()) def test_simple(self): @@ -286,6 +289,38 @@ return a, vTs # reused by a test in test_rtyper + def test_ll_stararg(self): + A = GcArray(Float) + B = GcArray(Signed) + def ll_sum(*args): + result = 0 + if len(args) > 0: + result += args[0] + if len(args) > 1: + result += args[1] + if len(args) > 2: + result += args[2] + if len(args) > 3: + result += args[3] + return result + def llf(): + a = ll_sum() + b = ll_sum(4, 5) + c = ll_sum(2.5) + d = ll_sum(4, 5.25) + e = ll_sum(1000, 200, 30, 4) + f = ll_sum(1000, 200, 30, 5) + return a, b, c, d, e, f + s = self.annotate(llf, []) + assert isinstance(s, annmodel.SomeTuple) + assert s.items[0].knowntype is int + assert s.items[0].const == 0 + assert s.items[1].knowntype is int + assert s.items[2].knowntype is float + assert s.items[3].knowntype is float + assert s.items[4].knowntype is int + assert s.items[5].knowntype is int + def test_getRuntimeTypeInfo(self): S = GcStruct('s', ('x', Signed)) attachRuntimeTypeInfo(S) From arigo at codespeak.net Thu Sep 20 13:22:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 13:22:54 +0200 (CEST) Subject: [pypy-svn] r46756 - pypy/dist/dotviewer Message-ID: <20070920112254.C2CB28196@code0.codespeak.net> Author: arigo Date: Thu Sep 20 13:22:53 2007 New Revision: 46756 Modified: pypy/dist/dotviewer/dotviewer.py pypy/dist/dotviewer/graphserver.py Log: Copy the pypy-dev mail about how to use graphserver.py into the docstrings. Modified: pypy/dist/dotviewer/dotviewer.py ============================================================================== --- pypy/dist/dotviewer/dotviewer.py (original) +++ pypy/dist/dotviewer/dotviewer.py Thu Sep 20 13:22:53 2007 @@ -10,7 +10,8 @@ In the second form, the graph was already compiled to a .plain file. In the third form, listen for connexion on the given port and display the graphs sent by the remote side. On the remote site, set the -GRAPHSERVER environment variable to HOST:PORT. +GRAPHSERVER environment variable to HOST:PORT. See graphserver.py +for more instructions. """ import sys Modified: pypy/dist/dotviewer/graphserver.py ============================================================================== --- pypy/dist/dotviewer/graphserver.py (original) +++ pypy/dist/dotviewer/graphserver.py Thu Sep 20 13:22:53 2007 @@ -1,9 +1,40 @@ #! /usr/bin/env python """ Usage: - graphserver.py + graphserver.py [interface:]port + dotviewer.py --server [interface:]port -Start a server listening for connexions on the given port. +Start a server listening for connexions on the given port. The two ways +to start a server are equivalent. The server displays the graphs sent +by a remote process. On the remote process' side, set the GRAPHSERVER +environment variable to HOST:PORT. + +Here is a step-by-step example on how to use it in combination with ssh +port forwarding (replace 9999 with a random port number of your choice, +e.g. between 8000 and 20000): + + - on your local machine, run: + graphserver.py localhost:9999 + + - connect to a remote machine, allowing the remote side to contact + your local dotviewer: + ssh -R9999:localhost:9999 machinename + + - set the $GRAPHSERVER env var on the remote machine to point to the + redirected port (in bash): + export GRAPHSERVER=:9999 + + - then any graph-viewing command you execute while $GRAPHSERVER is set + will reach your local dotviewer and basically appear to work + completely transparently. + + - to automate steps 2 and 3, I've put the following entry in my local + .ssh/config: + Host machinename + RemoteForward 9999 127.0.0.1:9999 + + and I've added the 'export GRAPHSERVER=:9999' line to my .bashrc on + the remote machine. """ import sys @@ -157,9 +188,8 @@ f = StringIO() traceback.print_exc(file=f) # try to add some explanations - help = (" | if you want to debug on a remote machine, set the\n" - " | GRAPHSERVER env var to a HOSTNAME:PORT pointing\n" - " | back to a locally running graphserver.py.") + help = (" | if you want to debug on a remote machine, see\n" + " | instructions in dotviewer/graphserver.py\n") try: import pygame except ImportError: From arigo at codespeak.net Thu Sep 20 13:34:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 13:34:32 +0200 (CEST) Subject: [pypy-svn] r46757 - in pypy/dist/pypy: annotation rlib rpython rpython/lltypesystem rpython/lltypesystem/test rpython/module rpython/test Message-ID: <20070920113432.BF4DA819B@code0.codespeak.net> Author: arigo Date: Thu Sep 20 13:34:32 2007 New Revision: 46757 Modified: pypy/dist/pypy/annotation/model.py pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/test/test_llann.py Log: Make 'stringpolicy' unnecessary. The wrapper around the external function is now more clever and gets specialized. It can detect the arguments that it is currently called with and adapts them only if needed. Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Thu Sep 20 13:34:32 2007 @@ -528,7 +528,11 @@ #____________________________________________________________ # annotation of low-level types +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.ootypesystem import ootype + class SomePtr(SomeObject): + knowntype = lltype._ptr immutable = True def __init__(self, ll_ptrtype): assert isinstance(ll_ptrtype, lltype.Ptr) @@ -566,9 +570,6 @@ def __init__(self, method): self.method = method -from pypy.rpython.lltypesystem import lltype -from pypy.rpython.ootypesystem import ootype - NUMBER = object() annotation_to_ll_map = [ (s_None, lltype.Void), # also matches SomeImpossibleValue() Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Thu Sep 20 13:34:32 2007 @@ -376,9 +376,7 @@ fd_set = cConfig.fd_set #c_int_size = sizeof(rffi.INT) -def external(*args, **kwds): - kwds.setdefault('stringpolicy', 'fullauto') - return rffi.llexternal(*args, **kwds) +external = rffi.llexternal if _POSIX: strerror = external('strerror', [rffi.INT], CCHARP) @@ -402,8 +400,7 @@ lltype.Ptr(rffi.CArray(addrinfo_ptr))], rffi.INT) freeaddrinfo = external('freeaddrinfo', [addrinfo_ptr], lltype.Void) getnameinfo = external('getnameinfo', [sockaddr_ptr, socklen_t, CCHARP, - size_t, CCHARP, size_t, rffi.INT], rffi.INT, - stringpolicy='noauto') + size_t, CCHARP, size_t, rffi.INT], rffi.INT) htonl = external('htonl', [rffi.UINT], rffi.UINT) htons = external('htons', [rffi.USHORT], rffi.USHORT) @@ -441,13 +438,12 @@ rffi.INT], ssize_t) recvfrom = external('recvfrom', [socketfd_type, rffi.VOIDP, size_t, rffi.INT, sockaddr_ptr, socklen_t_ptr], rffi.INT) -send = external('send', [socketfd_type, rffi.VOIDP, size_t, rffi.INT], +send = external('send', [socketfd_type, rffi.CCHARP, size_t, rffi.INT], ssize_t) sendto = external('sendto', [socketfd_type, rffi.VOIDP, size_t, rffi.INT, sockaddr_ptr, socklen_t], ssize_t) shutdown = external('shutdown', [socketfd_type, rffi.INT], rffi.INT) -gethostname = external('gethostname', [rffi.CCHARP, rffi.INT], rffi.INT, - stringpolicy='noauto') +gethostname = external('gethostname', [rffi.CCHARP, rffi.INT], rffi.INT) gethostbyname = external('gethostbyname', [rffi.CCHARP], lltype.Ptr(cConfig.hostent)) gethostbyaddr = external('gethostbyaddr', [rffi.VOIDP, rffi.INT, rffi.INT], lltype.Ptr(cConfig.hostent)) Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 13:34:32 2007 @@ -20,9 +20,8 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.lltypesystem.rffi import sizeof, offsetof -def castto(example, value): - return rffi.cast(lltype.typeOf(example), value) -castto._annspecialcase_ = 'specialize:ll' +def mallocbuf(buffersize): + return lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw') constants = _c.constants @@ -712,16 +711,18 @@ until at least one byte is available or until the remote end is closed. When the remote end is closed and all data is read, return the empty string.""" - read_bytes = -1 timeout = self._select(False) if timeout == 1: raise SocketTimeout elif timeout == 0: - buf = create_string_buffer(buffersize) - read_bytes = _c.socketrecv(self.fd, buf, buffersize, flags) - if read_bytes < 0: - raise self.error_handler() - return buf[:read_bytes] + buf = mallocbuf(buffersize) + try: + read_bytes = _c.socketrecv(self.fd, buf, buffersize, flags) + if read_bytes >= 0: + return buf[:read_bytes] + finally: + lltype.free(buf, flavor='raw') + raise self.error_handler() def recvfrom(self, buffersize, flags=0): """Like recv(buffersize, flags) but also return the sender's @@ -895,12 +896,15 @@ The arguments are the same as for socket() except the default family is AF_UNIX if defined on the platform; otherwise, the default is AF_INET. """ - result = _c.socketpair_t() - res = _c.socketpair(family, type, proto, byref(result)) + result = lltype.malloc(_c.socketpair_t, flavor='raw') + res = _c.socketpair(family, type, proto, result) if res < 0: raise last_error() - return (make_socket(result[0], family, type, proto, SocketClass), - make_socket(result[1], family, type, proto, SocketClass)) + fd0 = result[0] + fd1 = result[1] + lltype.free(result, flavor='raw') + return (make_socket(fd0, family, type, proto, SocketClass), + make_socket(fd1, family, type, proto, SocketClass)) if hasattr(_c, 'dup'): def fromfd(fd, family, type, proto=0, SocketClass=RSocket): Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Thu Sep 20 13:34:32 2007 @@ -116,9 +116,6 @@ kwds[item] = getattr(self, '_%s_' % item, []) else: kwds[item] += getattr(self, '_%s_' % item, []) - stringpolicy = getattr(self, '_stringpolicy_', None) - if stringpolicy and 'stringpolicy' not in kwds: - kwds['stringpolicy'] = stringpolicy return rffi.llexternal(*args, **kwds) def _freeze_(self): Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Sep 20 13:34:32 2007 @@ -28,14 +28,15 @@ def llexternal(name, args, result, _callable=None, sources=[], includes=[], libraries=[], include_dirs=[], sandboxsafe=False, - canraise=False, stringpolicy='noauto', _nowrapper=False): - """ String policies: - autocast - automatically cast to ll_string, but don't delete it - fullauto - automatically cast + delete is afterwards - noauto - don't do anything - - WARNING: It's likely that in future we'll decide to use fullauto by - default + canraise=False, _nowrapper=False): + """Build an external function that will invoke the C function 'name' + with the given 'args' types and 'result' type. + + You get by default a wrapper that casts between number types as needed + to match the arguments. You can also pass an RPython string when a + CCHARP argument is expected, and the C function receives a 'const char*' + pointing to a read-only null-terminated character of arrays, as usual + for C. """ ext_type = lltype.FuncType(args, result) if _callable is None: @@ -63,29 +64,29 @@ # when the test is fixed... assert isinstance(lltype.Signed, lltype.Number) real_args = () - if stringpolicy == 'fullauto': - to_free = () - for i, tp in unrolling_arg_tps: - ll_str = None - if isinstance(tp, lltype.Number): - real_args = real_args + (cast(tp, args[i]),) - elif tp is lltype.Float: - real_args = real_args + (float(args[i]),) - elif tp is CCHARP and (stringpolicy == 'fullauto' or - stringpolicy == 'autocast'): - s = args[i] - if s is None: - ll_str = lltype.nullptr(CCHARP.TO) - else: - ll_str = str2charp(s) - real_args = real_args + (ll_str,) + to_free = () + for i, TARGET in unrolling_arg_tps: + arg = args[i] + freeme = None + if TARGET == CCHARP: + if arg is None: + arg = lltype.nullptr(CCHARP.TO) # None => (char*)NULL + freeme = arg + elif isinstance(arg, str): + arg = str2charp(arg) + # XXX leaks if a str2charp() fails with MemoryError + # and was not the first in this function + freeme = arg else: - real_args = real_args + (args[i],) - if stringpolicy == 'fullauto': - if tp is CCHARP: - to_free = to_free + (ll_str,) - else: - to_free = to_free + (None,) + SOURCE = lltype.typeOf(arg) + if SOURCE != TARGET: + if TARGET is lltype.Float: + arg = float(arg) + elif (isinstance(SOURCE, lltype.Number) and + isinstance(TARGET, lltype.Number)): + arg = cast(TARGET, arg) + real_args = real_args + (arg,) + to_free = to_free + (freeme,) if invoke_around_handlers: before = aroundstate.before after = aroundstate.after @@ -95,12 +96,11 @@ result = funcptr(*real_args) if invoke_around_handlers: if after: after() - if stringpolicy == 'fullauto': - for i, tp in unrolling_arg_tps: - if tp is CCHARP: - if to_free[i]: - lltype.free(to_free[i], flavor='raw') + for i, TARGET in unrolling_arg_tps: + if to_free[i]: + lltype.free(to_free[i], flavor='raw') return result + wrapper._annspecialcase_ = 'specialize:ll' wrapper._always_inline_ = True # for debugging, stick ll func ptr to that wrapper._ptr = funcptr Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Thu Sep 20 13:34:32 2007 @@ -45,8 +45,7 @@ assert xf() == 8+3 def test_string(): - z = llexternal('strlen', [CCHARP], Signed, includes=['string.h'], - stringpolicy='noauto') + z = llexternal('strlen', [CCHARP], Signed, includes=['string.h']) def f(): s = str2charp("xxx") @@ -69,8 +68,7 @@ return ret; } """) - z = llexternal('f', [CCHARP], CCHARP, sources=[c_source], - stringpolicy='noauto') + z = llexternal('f', [CCHARP], CCHARP, sources=[c_source]) def f(): s = str2charp("xxx") @@ -376,13 +374,13 @@ def test_stringpolicy1(): - strlen = llexternal('strlen', [CCHARP], INT, includes=['string.h'], - stringpolicy='fullauto') + strlen = llexternal('strlen', [CCHARP], INT, includes=['string.h']) def f(): return strlen("Xxx") assert interpret(f, [], backendopt=True) == 3 def test_stringpolicy2(): + py.test.skip("stringpolicy='autocast' no longer implemented") def f(): return strlen("Xxx") strlen = llexternal('strlen', [CCHARP], INT, @@ -390,8 +388,7 @@ py.test.raises(MallocMismatch, interpret, f, [], backendopt=True) def test_stringpolicy3(): - strlen = llexternal('strlen', [CCHARP], INT, - includes=['string.h'], stringpolicy='noauto') + strlen = llexternal('strlen', [CCHARP], INT, includes=['string.h']) def f(): ll_str = str2charp("Xxx") res = strlen(ll_str) @@ -399,7 +396,19 @@ return res assert interpret(f, [], backendopt=True) == 3 - + +def test_stringpolicy_mixed(): + strlen = llexternal('strlen', [CCHARP], INT, + includes=['string.h']) + def f(): + res1 = strlen("abcd") + ll_str = str2charp("Xxx") + res2 = strlen(ll_str) + lltype.free(ll_str, flavor='raw') + return res1*10 + res2 + + assert interpret(f, [], backendopt=True) == 43 + def test_around_extcall(): import os from pypy.annotation import model as annmodel Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Thu Sep 20 13:34:32 2007 @@ -63,8 +63,7 @@ class RegisterOs(BaseLazyRegistering): - _stringpolicy_ = 'fullauto' - + def __init__(self): self.configure(CConfig) @@ -519,7 +518,7 @@ def register_os_getcwd(self): os_getcwd = self.llexternal(underscore_on_windows + 'getcwd', [rffi.CCHARP, rffi.SIZE_T], - rffi.CCHARP, stringpolicy='noauto') + rffi.CCHARP) def os_getcwd_llimpl(): bufsize = 256 @@ -710,7 +709,7 @@ def register_os_readlink(self): os_readlink = self.llexternal('readlink', [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], - rffi.INT, stringpolicy='noauto') + rffi.INT) # XXX SSIZE_T in POSIX.1-2001 def os_readlink_llimpl(path): Modified: pypy/dist/pypy/rpython/test/test_llann.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llann.py (original) +++ pypy/dist/pypy/rpython/test/test_llann.py Thu Sep 20 13:34:32 2007 @@ -321,6 +321,24 @@ assert s.items[4].knowntype is int assert s.items[5].knowntype is int + def test_str_vs_ptr(self): + S = GcStruct('s', ('x', Signed)) + def ll_stuff(x): + if x is None or isinstance(x, str): + return 2 + else: + return 3 + def llf(): + x = ll_stuff("hello") + y = ll_stuff(nullptr(S)) + return x, y + s = self.annotate(llf, []) + assert isinstance(s, annmodel.SomeTuple) + assert s.items[0].is_constant() + assert s.items[0].const == 2 + assert s.items[1].is_constant() + assert s.items[1].const == 3 + def test_getRuntimeTypeInfo(self): S = GcStruct('s', ('x', Signed)) attachRuntimeTypeInfo(S) From arigo at codespeak.net Thu Sep 20 13:36:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 13:36:59 +0200 (CEST) Subject: [pypy-svn] r46758 - in pypy/dist/pypy/rlib: . test Message-ID: <20070920113659.36F78819E@code0.codespeak.net> Author: arigo Date: Thu Sep 20 13:36:58 2007 New Revision: 46758 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: Fix recv(). Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 13:36:58 2007 @@ -719,7 +719,8 @@ try: read_bytes = _c.socketrecv(self.fd, buf, buffersize, flags) if read_bytes >= 0: - return buf[:read_bytes] + assert read_bytes <= buffersize + return ''.join([buf[i] for i in range(read_bytes)]) finally: lltype.free(buf, flavor='raw') raise self.error_handler() Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 13:36:58 2007 @@ -44,7 +44,6 @@ assert a.get_host() == "127.0.0.1" def test_socketpair(): - py.test.skip("in-progress") if sys.platform == "win32": py.test.skip('No socketpair on Windows') s1, s2 = socketpair() From arigo at codespeak.net Thu Sep 20 14:28:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 14:28:07 +0200 (CEST) Subject: [pypy-svn] r46760 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070920122807.7AA1481A3@code0.codespeak.net> Author: arigo Date: Thu Sep 20 14:28:07 2007 New Revision: 46760 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Implement errno support in ll2ctypes. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Thu Sep 20 14:28:07 2007 @@ -11,6 +11,7 @@ from pypy.rpython.extfunc import ExtRegistryEntry from pypy.rlib.objectmodel import Symbolic from pypy.tool.uid import fixid +from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint from pypy.annotation import model as annmodel from pypy.rpython.rbuiltin import gen_cast @@ -534,7 +535,9 @@ RESULT = FUNCTYPE.RESULT def invoke_via_ctypes(*argvalues): cargs = [lltype2ctypes(value) for value in argvalues] + _restore_c_errno() cres = cfunc(*cargs) + _save_c_errno() return ctypes2lltype(RESULT, cres) return invoke_via_ctypes @@ -578,3 +581,43 @@ v_arg = hop.inputarg(hop.args_r[1], arg=1) TYPE1 = v_arg.concretetype return gen_cast(hop.llops, RESTYPE, v_arg) + +# ____________________________________________________________ +# errno + +# this saves in a thread-local way the "current" value that errno +# should have in C. We have to save it away from one external C function +# call to the next. Otherwise a non-zero value left behind will confuse +# CPython itself a bit later, and/or CPython will stamp on it before we +# try to inspect it via rffi.get_errno(). +TLS = tlsobject() + +# helpers to save/restore the C-level errno -- platform-specific because +# ctypes doesn't just do the right thing and expose it directly :-( +def _where_is_errno(): + raise NotImplementedError("don't know how to get the C-level errno!") + +def _save_c_errno(): + errno_p = _where_is_errno() + TLS.errno = errno_p.contents.value + errno_p.contents.value = 0 + +def _restore_c_errno(): + if hasattr(TLS, 'errno'): + _where_is_errno().contents.value = TLS.errno + +if ctypes: + if sys.platform == 'win32': + standard_c_lib._errno.restype = ctypes.POINTER(ctypes.c_int) + def _where_is_errno(): + return standard_c_lib._errno() + + elif sys.platform in ('linux2', 'freebsd6'): + standard_c_lib.__errno_location.restype = ctypes.POINTER(ctypes.c_int) + def _where_is_errno(): + return standard_c_lib.__errno_location() + + elif sys.platform == 'darwin': + standard_c_lib.__errno.restype = ctypes.POINTER(ctypes.c_int) + def _where_is_errno(): + return standard_c_lib.__errno() Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Sep 20 14:28:07 2007 @@ -211,7 +211,7 @@ def COpaquePtr(*args, **kwds): return lltype.Ptr(COpaque(*args, **kwds)) -def CExternVariable(TYPE, name): +def CExternVariable(TYPE, name, _CConstantClass=CConstant): """Return a pair of functions - a getter and a setter - to access the given global C variable. """ @@ -221,7 +221,7 @@ # some #include... assert not isinstance(TYPE, lltype.ContainerType) CTYPE = lltype.FixedSizeArray(TYPE, 1) - c_variable_ref = CConstant('(&%s)' % (name,), lltype.Ptr(CTYPE)) + c_variable_ref = _CConstantClass('(&%s)' % (name,), lltype.Ptr(CTYPE)) def getter(): return c_variable_ref[0] def setter(newvalue): @@ -229,7 +229,22 @@ return (func_with_new_name(getter, '%s_getter' % (name,)), func_with_new_name(setter, '%s_setter' % (name,))) -get_errno, set_errno = CExternVariable(lltype.Signed, 'errno') + +class CConstantErrno(CConstant): + # these accessors are used when calling get_errno() or set_errno() + # on top of CPython + def __getitem__(self, index): + assert index == 0 + try: + return ll2ctypes.TLS.errno + except AttributeError: + raise ValueError("no C function call occurred so far, " + "errno is undefined") + def __setitem__(self, index, value): + assert index == 0 + ll2ctypes.TLS.errno = value + +get_errno, set_errno = CExternVariable(lltype.Signed, 'errno', CConstantErrno) # char, represented as a Python character # (use SIGNEDCHAR or UCHAR for the small integer types) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu Sep 20 14:28:07 2007 @@ -516,3 +516,24 @@ lltype.free(s1, flavor='raw') lltype.free(a2, flavor='raw') lltype.free(s2, flavor='raw') + + def test_get_errno(self): + if sys.platform.startswith('win'): + underscore_on_windows = '_' + else: + underscore_on_windows = '' + strlen = rffi.llexternal('strlen', [rffi.CCHARP], rffi.SIZE_T, + includes=['string.h']) + os_write = rffi.llexternal(underscore_on_windows+'write', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], + rffi.SIZE_T) + buffer = lltype.malloc(rffi.CCHARP.TO, 5, flavor='raw') + written = os_write(12312312, buffer, 5) + lltype.free(buffer, flavor='raw') + assert rffi.cast(lltype.Signed, written) < 0 + # the next line is a random external function call, + # to check that it doesn't reset errno + strlen("hi!") + err = rffi.get_errno() + import errno + assert err == errno.EBADF From arigo at codespeak.net Thu Sep 20 14:57:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 14:57:40 +0200 (CEST) Subject: [pypy-svn] r46761 - in pypy/dist/pypy/rlib: . test Message-ID: <20070920125740.858688191@code0.codespeak.net> Author: arigo Date: Thu Sep 20 14:57:40 2007 New Revision: 46761 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: Some more operations. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Thu Sep 20 14:57:40 2007 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rffi import CCHARP +from pypy.rpython.lltypesystem.rffi import get_errno as geterrno from pypy.rlib.rarithmetic import intmask, r_uint import os @@ -379,7 +380,6 @@ external = rffi.llexternal if _POSIX: - strerror = external('strerror', [rffi.INT], CCHARP) gai_strerror = external('gai_strerror', [rffi.INT], CCHARP) #h_errno = c_int.in_dll(socketdll, 'h_errno') @@ -421,10 +421,11 @@ socklen_t], CCHARP) inet_addr = external('inet_addr', [rffi.CCHARP], rffi.UINT) -socklen_t_ptr = rffi.CArray(socklen_t) +socklen_t_ptr = lltype.Ptr(rffi.CFixedArray(socklen_t, 1)) socketaccept = external('accept', [socketfd_type, sockaddr_ptr, socklen_t_ptr], rffi.INT) -bind = external('bind', [rffi.INT, socketfd_type, socklen_t], rffi.INT) +socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t], + rffi.INT) socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT) socketgetpeername = external('getpeername', [socketfd_type, sockaddr_ptr, socklen_t_ptr], rffi.INT) @@ -582,5 +583,4 @@ def socket_strerror(errno): return WIN32_ERROR_MESSAGES.get(errno, "winsock error %d" % errno) else: - def socket_strerror(errno): - return strerror(errno) + socket_strerror = os.strerror Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 14:57:40 2007 @@ -456,12 +456,10 @@ def make_null_address(family): klass = familyclass(family) - buf = create_string_buffer(klass.maxlen) result = instantiate(klass) - XXX; result._addr_keepalive2 = buf - result.addr = cast(buf, _c.sockaddr_ptr).contents - result.addrlen = 0 - return result, len(buf) + buf = mallocbuf(klass.maxlen) + result.setdata(buf, 0) + return result, klass.maxlen def ipaddr_from_object(space, w_sockaddr): host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) @@ -562,7 +560,9 @@ def _addrbuf(self): addr, maxlen = make_null_address(self.family) - return addr, _c.socklen_t(maxlen) + addrlen_p = lltype.malloc(_c.socklen_t_ptr.TO, flavor='raw') + addrlen_p[0] = rffi.cast(_c.socklen_t, maxlen) + return addr, addrlen_p def accept(self, SocketClass=None): """Wait for an incoming connection. @@ -571,18 +571,22 @@ SocketClass = RSocket if self._select(False) == 1: raise SocketTimeout - address, addrlen = self._addrbuf() - newfd = _c.socketaccept(self.fd, byref(address.addr), byref(addrlen)) + address, addrlen_p = self._addrbuf() + try: + newfd = _c.socketaccept(self.fd, address.addr, addrlen_p) + addrlen = addrlen_p[0] + finally: + lltype.free(addrlen_p, flavor='raw') if _c.invalid_socket(newfd): raise self.error_handler() - address.addrlen = addrlen.value + address.addrlen = addrlen sock = make_socket(newfd, self.family, self.type, self.proto, SocketClass) return (sock, address) def bind(self, address): """Bind the socket to a local address.""" - res = _c.socketbind(self.fd, byref(address.addr), address.addrlen) + res = _c.socketbind(self.fd, address.addr, address.addrlen) if res < 0: raise self.error_handler() @@ -597,13 +601,13 @@ def connect(self, address): """Connect the socket to a remote address.""" - res = _c.socketconnect(self.fd, byref(address.addr), address.addrlen) + res = _c.socketconnect(self.fd, address.addr, address.addrlen) if self.timeout > 0.0: errno = _c.geterrno() if res < 0 and errno == _c.EINPROGRESS: timeout = self._select(True) if timeout == 0: - res = _c.socketconnect(self.fd, byref(address.addr), + res = _c.socketconnect(self.fd, address.addr, address.addrlen) elif timeout == -1: raise self.error_handler() @@ -616,13 +620,13 @@ def connect_ex(self, address): """This is like connect(address), but returns an error code (the errno value) instead of raising an exception when an error occurs.""" - res = _c.socketconnect(self.fd, byref(address.addr), address.addrlen) + res = _c.socketconnect(self.fd, address.addr, address.addrlen) if self.timeout > 0.0: errno = _c.geterrno() if res < 0 and errno == _c.EINPROGRESS: timeout = self._select(True) if timeout == 0: - res = _c.socketconnect(self.fd, byref(address.addr), + res = _c.socketconnect(self.fd, address.addr, address.addrlen) elif timeout == -1: return _c.geterrno() @@ -651,22 +655,28 @@ def getpeername(self): """Return the address of the remote endpoint.""" - address, addrlen = self._addrbuf() - res = _c.socketgetpeername(self.fd, byref(address.addr), - byref(addrlen)) + address, addrlen_p = self._addrbuf() + try: + res = _c.socketgetpeername(self.fd, address.addr, addrlen_p) + addrlen = addrlen_p[0] + finally: + lltype.free(addrlen_p, flavor='raw') if res < 0: raise self.error_handler() - address.addrlen = addrlen.value + address.addrlen = addrlen return address def getsockname(self): """Return the address of the local endpoint.""" - address, addrlen = self._addrbuf() - res = _c.socketgetsockname(self.fd, byref(address.addr), - byref(addrlen)) + address, addrlen_p = self._addrbuf() + try: + res = _c.socketgetsockname(self.fd, address.addr, addrlen_p) + addrlen = addrlen_p[0] + finally: + lltype.free(addrlen_p, flavor='raw') if res < 0: raise self.error_handler() - address.addrlen = addrlen.value + address.addrlen = addrlen return address def getsockopt(self, level, option, maxlen): @@ -733,18 +743,25 @@ if timeout == 1: raise SocketTimeout elif timeout == 0: - buf = create_string_buffer(buffersize) - address, addrlen = self._addrbuf() - read_bytes = _c.recvfrom(self.fd, buf, buffersize, flags, - byref(address.addr), byref(addrlen)) - if read_bytes < 0: - raise self.error_handler() - result_addrlen = addrlen.value - if result_addrlen: - address.addrlen = result_addrlen - else: - address = None - return (buf[:read_bytes], address) + buf = mallocbuf(buffersize) + try: + address, addrlen_p = self._addrbuf() + try: + read_bytes = _c.recvfrom(self.fd, buf, buffersize, flags, + address.addr, addrlen_p) + addrlen = addrlen_p[0] + finally: + lltype.free(addrlen_p, flavor='raw') + if read_bytes >= 0: + if addrlen: + address.addrlen = addrlen + else: + address = None + data = ''.join([buf[i] for i in range(read_bytes)]) + return (data, address) + finally: + lltype.free(buf, flavor='raw') + raise self.error_handler() def send(self, data, flags=0): """Send a data string to the socket. For the optional flags Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 14:57:40 2007 @@ -58,7 +58,6 @@ s2.close() def test_simple_tcp(): - py.test.skip("in-progress") import thread sock = RSocket() try_ports = [1023] + range(20000, 30000, 437) @@ -78,21 +77,27 @@ sock.listen(1) s2 = RSocket(AF_INET, SOCK_STREAM) thread.start_new_thread(s2.connect, (addr,)) + print 'waiting for connexion' s1, addr2 = sock.accept() + print 'connexion accepted' assert addr.eq(s2.getpeername()) assert addr2.eq(s2.getsockname()) assert addr2.eq(s1.getpeername()) s1.send('?') + print 'sent one character' buf = s2.recv(100) assert buf == '?' - thread.start_new_thread(s2.sendall, ('x'*500000,)) + print 'received ok' + thread.start_new_thread(s2.sendall, ('x'*50000,)) buf = '' - while len(buf) < 500000: - data = s1.recv(500100) + while len(buf) < 50000: + data = s1.recv(50100) + print 'recv returned %d bytes' % (len(data,)) assert data buf += data - assert buf == 'x'*500000 + assert buf == 'x'*50000 + print 'data received ok' s1.close() s2.close() From arigo at codespeak.net Thu Sep 20 15:01:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 15:01:22 +0200 (CEST) Subject: [pypy-svn] r46762 - in pypy/dist/pypy/rlib: . test Message-ID: <20070920130122.21EDA8193@code0.codespeak.net> Author: arigo Date: Thu Sep 20 15:01:20 2007 New Revision: 46762 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: Some more operations. Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 15:01:20 2007 @@ -795,7 +795,7 @@ raise SocketTimeout elif timeout == 0: res = _c.sendto(self.fd, data, len(data), flags, - byref(address.addr), address.addrlen) + address.addr, address.addrlen) if res < 0: raise self.error_handler() return res @@ -1025,7 +1025,7 @@ addr = make_address(info.c_ai_addr, info.c_ai_addrlen, address_to_fill) if info.c_ai_canonname: - canonname = charp2str(info.c_ai_canonname) + canonname = rffi.charp2str(info.c_ai_canonname) else: canonname = "" result.append((info.c_ai_family, Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 15:01:20 2007 @@ -102,7 +102,6 @@ s2.close() def test_simple_udp(): - py.test.skip("in-progress") s1 = RSocket(AF_INET, SOCK_DGRAM) try_ports = [1023] + range(20000, 30000, 437) for port in try_ports: @@ -134,7 +133,6 @@ s2.close() def test_nonblocking(): - py.test.skip("in-progress") sock = RSocket() sock.setblocking(False) try_ports = [1023] + range(20000, 30000, 437) @@ -176,8 +174,8 @@ assert buf == '?' err = py.test.raises(CSocketError, s1.recv, 5000) assert err.value.errno in (errno.EAGAIN, errno.EWOULDBLOCK) - count = s2.send('x'*500000) - assert 1 <= count <= 500000 + count = s2.send('x'*50000) + assert 1 <= count <= 50000 while count: # Recv may return less than requested buf = s1.recv(count + 100) assert len(buf) <= count @@ -211,7 +209,6 @@ assert found, lst def test_getaddrinfo_no_reverse_lookup(): - py.test.skip("in-progress") # It seems that getaddrinfo never runs a reverse lookup on Linux. # Python2.3 on Windows returns the hostname. lst = getaddrinfo('134.99.112.214', None, flags=AI_CANONNAME) @@ -224,7 +221,6 @@ assert found, lst def test_connect_ex(): - py.test.skip("in-progress") s = RSocket() err = s.connect_ex(INETAddress('0.0.0.0', 0)) # should not work assert err in (errno.ECONNREFUSED, errno.EADDRNOTAVAIL) From arigo at codespeak.net Thu Sep 20 15:14:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 15:14:21 +0200 (CEST) Subject: [pypy-svn] r46763 - in pypy/dist/pypy/rlib: . test Message-ID: <20070920131421.775FE8171@code0.codespeak.net> Author: arigo Date: Thu Sep 20 15:14:19 2007 New Revision: 46763 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: getsockopt(), setsockopt() Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 15:14:19 2007 @@ -680,25 +680,41 @@ return address def getsockopt(self, level, option, maxlen): - buf = _c.create_string_buffer(maxlen) - bufsize = _c.socklen_t() - bufsize.value = maxlen - res = _c.socketgetsockopt(self.fd, level, option, cast(buf, POINTER(c_char)), byref(bufsize)) - if res < 0: - raise self.error_handler() - size = bufsize.value - assert size > 0 # socklen_t is signed on Windows - return buf.raw[:size] + buf = mallocbuf(maxlen) + try: + bufsize_p = lltype.malloc(_c.socklen_t_ptr.TO, flavor='raw') + try: + bufsize_p[0] = rffi.cast(_c.socklen_t, maxlen) + res = _c.socketgetsockopt(self.fd, level, option, + buf, bufsize_p) + if res < 0: + raise self.error_handler() + size = bufsize_p[0] + assert size >= 0 # socklen_t is signed on Windows + result = ''.join([buf[i] for i in range(size)]) + finally: + lltype.free(bufsize_p, flavor='raw') + finally: + lltype.free(buf, flavor='raw') + return result def getsockopt_int(self, level, option): - flag = _c.c_int() - flagsize = _c.socklen_t() - flagsize.value = sizeof(flag) - res = _c.socketgetsockopt(self.fd, level, option, - byref(flag), byref(flagsize)) - if res < 0: - raise self.error_handler() - return flag.value + flag_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + try: + flagsize_p = lltype.malloc(_c.socklen_t_ptr.TO, flavor='raw') + try: + flagsize_p[0] = rffi.cast(_c.socklen_t, rffi.sizeof(rffi.INT)) + res = _c.socketgetsockopt(self.fd, level, option, + rffi.cast(rffi.VOIDP, flag_p), + flagsize_p) + if res < 0: + raise self.error_handler() + result = flag_p[0] + finally: + lltype.free(flagsize_p, flavor='raw') + finally: + lltype.free(flag_p, flavor='raw') + return result def gettimeout(self): """Return the timeout of the socket. A timeout < 0 means that @@ -808,14 +824,19 @@ self.settimeout(timeout) def setsockopt(self, level, option, value): - res = _c.socketsetsockopt(self.fd, level, option, c_char_p(value), len(value)) + res = _c.socketsetsockopt(self.fd, level, option, value, len(value)) if res < 0: raise self.error_handler() def setsockopt_int(self, level, option, value): - flag = _c.c_int(value) - res = _c.socketsetsockopt(self.fd, level, option, - byref(flag), sizeof(flag)) + flag_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + try: + flag_p[0] = rffi.cast(rffi.INT, value) + res = _c.socketsetsockopt(self.fd, level, option, + rffi.cast(rffi.VOIDP, flag_p), + rffi.sizeof(rffi.INT)) + finally: + lltype.free(flag_p, flavor='raw') if res < 0: raise self.error_handler() Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 15:14:19 2007 @@ -227,8 +227,8 @@ def test_getsetsockopt(): - py.test.skip("rewrite without ctypes") - from ctypes import c_int, c_char, c_char_p, POINTER, cast, pointer, sizeof + import struct + assert struct.calcsize("i") == rffi.sizeof(rffi.INT) # A socket sould start with reuse == 0 s = RSocket(AF_INET, SOCK_STREAM) reuse = s.getsockopt_int(SOL_SOCKET, SO_REUSEADDR) @@ -238,22 +238,14 @@ assert reuse != 0 # Test string case s = RSocket(AF_INET, SOCK_STREAM) - reusestr = s.getsockopt(SOL_SOCKET, SO_REUSEADDR, sizeof(c_int)) - # XXX: This strange creation fo reuse_c_char instead of plain - # c_char_p(reusestr) is to work around a bug in the cast function - # of ctypes version 1.0.0 - reuse_c_chars = (c_char*len(reusestr))(*[c for c in reusestr]) - reuseptr = cast(reuse_c_chars, POINTER(c_int)) - assert reuseptr[0] == 0 - optval = c_int(1) - optvalp = cast(pointer(optval), POINTER(c_char)) - optstr = optvalp[:sizeof(c_int)] + reusestr = s.getsockopt(SOL_SOCKET, SO_REUSEADDR, rffi.sizeof(rffi.INT)) + value, = struct.unpack("i", reusestr) + assert value == 0 + optstr = struct.pack("i", 1) s.setsockopt(SOL_SOCKET, SO_REUSEADDR, optstr) - reusestr = s.getsockopt(SOL_SOCKET, SO_REUSEADDR, sizeof(c_int)) - # XXX: See above. - reuse_c_chars = (c_char*len(reusestr))(*[c for c in reusestr]) - reuseptr = cast(reuse_c_chars, POINTER(c_int)) - assert reuseptr[0] != 0 + reusestr = s.getsockopt(SOL_SOCKET, SO_REUSEADDR, rffi.sizeof(rffi.INT)) + value, = struct.unpack("i", reusestr) + assert value != 0 def test_dup(): py.test.skip("in-progress") From arigo at codespeak.net Thu Sep 20 15:15:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 15:15:54 +0200 (CEST) Subject: [pypy-svn] r46764 - in pypy/dist/pypy/rlib: . test Message-ID: <20070920131554.66251817E@code0.codespeak.net> Author: arigo Date: Thu Sep 20 15:15:54 2007 New Revision: 46764 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: socket().dup(). Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Thu Sep 20 15:15:54 2007 @@ -380,6 +380,7 @@ external = rffi.llexternal if _POSIX: + dup = external('dup', [socketfd_type], socketfd_type) gai_strerror = external('gai_strerror', [rffi.INT], CCHARP) #h_errno = c_int.in_dll(socketdll, 'h_errno') Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 15:15:54 2007 @@ -248,7 +248,6 @@ assert value != 0 def test_dup(): - py.test.skip("in-progress") if sys.platform == "win32": skip("dup does not work on Windows") s = RSocket(AF_INET, SOCK_STREAM) From arigo at codespeak.net Thu Sep 20 15:32:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 15:32:35 +0200 (CEST) Subject: [pypy-svn] r46765 - in pypy/dist/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20070920133235.BDB18818F@code0.codespeak.net> Author: arigo Date: Thu Sep 20 15:32:33 2007 New Revision: 46765 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: inet_aton() and maybe inet_ntoa() but the latter is not tested Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 15:32:33 2007 @@ -1097,34 +1097,47 @@ if hasattr(_c, 'inet_aton'): def inet_aton(ip): "IPv4 dotted string -> packed 32-bits string" - buf = create_string_buffer(sizeof(_c.in_addr)) - if _c.inet_aton(ip, cast(buf, POINTER(_c.in_addr))): - return buf.raw - else: - raise RSocketError("illegal IP address string passed to inet_aton") + size = sizeof(_c.in_addr) + buf = mallocbuf(size) + try: + if _c.inet_aton(ip, rffi.cast(lltype.Ptr(_c.in_addr), buf)): + return ''.join([buf[i] for i in range(size)]) + else: + raise RSocketError("illegal IP address string passed to inet_aton") + finally: + lltype.free(buf, flavor='raw') else: def inet_aton(ip): "IPv4 dotted string -> packed 32-bits string" if ip == "255.255.255.255": return "\xff\xff\xff\xff" packed_addr = _c.inet_addr(ip) - if _c.c_long(packed_addr).value == INADDR_NONE: + if packed_addr == rffi.cast(rffi.UINT, INADDR_NONE): raise RSocketError("illegal IP address string passed to inet_aton") - buf = copy_buffer(cast(pointer(c_ulong(packed_addr)), - POINTER(c_char)), 4) - return buf.raw + size = sizeof(_c.in_addr) + buf = mallocbuf(size) + try: + rffi.cast(rffi.UINTP, buf)[0] = packed_addr + return ''.join([buf[i] for i in range(size)]) + finally: + lltype.free(buf, flavor='raw') def inet_ntoa(packed): "packet 32-bits string -> IPv4 dotted string" if len(packed) != sizeof(_c.in_addr): raise RSocketError("packed IP wrong length for inet_ntoa") - buf = create_string_buffer(sizeof(_c.in_addr)) - buf.raw = packed - return _c.inet_ntoa(cast(buf, POINTER(_c.in_addr)).contents) + buf = rffi.make(_c.in_addr) + try: + for i in range(sizeof(_c.in_addr)): + rffi.cast(rffi.CCHARP, buf)[i] = packed[i] + return _c.inet_ntoa(buf) + finally: + lltype.free(buf, flavor='raw') if hasattr(_c, 'inet_pton'): def inet_pton(family, ip): "human-readable string -> packed string" + XXX if family == AF_INET: size = sizeof(_c.in_addr) elif AF_INET6 is not None and family == AF_INET6: @@ -1143,6 +1156,7 @@ if hasattr(_c, 'inet_ntop'): def inet_ntop(family, packed): "packed string -> human-readable string" + XXX if family == AF_INET: srcsize = sizeof(_c.in_addr) dstsize = _c.INET_ADDRSTRLEN Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 15:32:33 2007 @@ -258,7 +258,6 @@ assert s.getsockname().eq(s2.getsockname()) def test_inet_aton(): - py.test.skip("in-progress") assert inet_aton('1.2.3.4') == '\x01\x02\x03\x04' assert inet_aton('127.0.0.1') == '\x7f\x00\x00\x01' tests = ["127.0.0.256", "127.0.0.255555555555555555", "127.2b.0.0", Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Sep 20 15:32:33 2007 @@ -147,6 +147,8 @@ tp = platform.inttype(name.upper(), c_name, signed) globals()['r_' + name] = platform.numbertype_to_rclass[tp] globals()[name.upper()] = tp + tpp = lltype.Ptr(lltype.Array(tp, hints={'nolength': True})) + globals()[name.upper()+'P'] = tpp result.append(tp) return result @@ -259,8 +261,8 @@ # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) -# int * -INTP = lltype.Ptr(lltype.Array(lltype.Signed, hints={'nolength': True})) +# int *, unsigned int *, etc. +#INTP = ... see setup() above # double * DOUBLEP = lltype.Ptr(lltype.Array(DOUBLE, hints={'nolength': True})) From arigo at codespeak.net Thu Sep 20 15:47:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 15:47:17 +0200 (CEST) Subject: [pypy-svn] r46766 - in pypy/dist/pypy: rlib rlib/test rpython/lltypesystem rpython/lltypesystem/test Message-ID: <20070920134717.DFE06817E@code0.codespeak.net> Author: arigo Date: Thu Sep 20 15:47:16 2007 New Revision: 46766 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: Test and fix inet_ntoa(). Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 15:47:16 2007 @@ -1130,7 +1130,7 @@ try: for i in range(sizeof(_c.in_addr)): rffi.cast(rffi.CCHARP, buf)[i] = packed[i] - return _c.inet_ntoa(buf) + return rffi.charp2str(_c.inet_ntoa(buf)) finally: lltype.free(buf, flavor='raw') Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 15:47:16 2007 @@ -272,7 +272,10 @@ assert inet_aton(ip) == aton except SocketError: pass - + +def test_inet_ntoa(): + assert inet_ntoa('\x01\x02\x03\x04') == '1.2.3.4' + class TestTCP: PORT = 50007 HOST = 'localhost' Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Thu Sep 20 15:47:16 2007 @@ -533,8 +533,14 @@ def get_ctypes_trampoline(FUNCTYPE, cfunc): RESULT = FUNCTYPE.RESULT + container_arguments = [] + for i in range(len(FUNCTYPE.ARGS)): + if isinstance(FUNCTYPE.ARGS[i], lltype.ContainerType): + container_arguments.append(i) def invoke_via_ctypes(*argvalues): cargs = [lltype2ctypes(value) for value in argvalues] + for i in container_arguments: + cargs[i] = cargs[i].contents _restore_c_errno() cres = cfunc(*cargs) _save_c_errno() Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Thu Sep 20 15:47:16 2007 @@ -1100,8 +1100,14 @@ raise TypeError,"calling %r with wrong argument number: %r" % (self._T, args) for a, ARG in zip(args, self._T.ARGS): if typeOf(a) != ARG: - args_repr = [typeOf(arg) for arg in args] - raise TypeError,"calling %r with wrong argument types: %r" % (self._T, args_repr) + # special case: ARG can be a container type, in which + # case a should be a pointer to it. This must also be + # special-cased in the backends. + if not (isinstance(ARG, ContainerType) + and typeOf(a) == Ptr(ARG)): + args_repr = [typeOf(arg) for arg in args] + raise TypeError, ("calling %r with wrong argument " + "types: %r" % (self._T, args_repr)) callb = self._obj._callable if callb is None: raise RuntimeError,"calling undefined function" Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu Sep 20 15:47:16 2007 @@ -537,3 +537,15 @@ err = rffi.get_errno() import errno assert err == errno.EBADF + + def test_call_with_struct_argument(self): + # XXX is there such a function in the standard C headers? + from pypy.rlib import _rsocket_rffi + buf = rffi.make(_rsocket_rffi.in_addr) + rffi.cast(rffi.CCHARP, buf)[0] = '\x01' + rffi.cast(rffi.CCHARP, buf)[1] = '\x02' + rffi.cast(rffi.CCHARP, buf)[2] = '\x03' + rffi.cast(rffi.CCHARP, buf)[3] = '\x04' + p = _rsocket_rffi.inet_ntoa(buf) + assert rffi.charp2str(p) == '1.2.3.4' + lltype.free(buf, flavor='raw') Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Thu Sep 20 15:47:16 2007 @@ -124,9 +124,9 @@ return (z->one + z->three); } """ - TP = CStructPtr('xx', ('one', Signed), ('two', Char), ('three', Signed)) + TP = CStructPtr('xx', ('one', INT), ('two', Char), ('three', INT)) - z = llexternal('f', [TP], Signed, sources=[c_source], + z = llexternal('f', [TP], INT, sources=[c_source], includes=[str(h_file)], include_dirs=[udir]) def f(): From arigo at codespeak.net Thu Sep 20 15:57:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 15:57:45 +0200 (CEST) Subject: [pypy-svn] r46767 - in pypy/dist/pypy/rlib: . test Message-ID: <20070920135745.036C8817D@code0.codespeak.net> Author: arigo Date: Thu Sep 20 15:57:41 2007 New Revision: 46767 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: inet_pton(), inet_ntop(), with minimal tests for them. Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 15:57:41 2007 @@ -1137,26 +1137,28 @@ if hasattr(_c, 'inet_pton'): def inet_pton(family, ip): "human-readable string -> packed string" - XXX if family == AF_INET: size = sizeof(_c.in_addr) elif AF_INET6 is not None and family == AF_INET6: size = sizeof(_c.in6_addr) else: raise RSocketError("unknown address family") - buf = create_string_buffer(size) - res = _c.inet_pton(family, ip, cast(buf, c_void_p)) - if res < 0: - raise last_error() - elif res == 0: - raise RSocketError("illegal IP address string passed to inet_pton") - else: - return buf.raw + buf = mallocbuf(size) + try: + res = _c.inet_pton(family, ip, buf) + if res < 0: + raise last_error() + elif res == 0: + raise RSocketError("illegal IP address string passed " + "to inet_pton") + else: + return ''.join([buf[i] for i in range(size)]) + finally: + lltype.free(buf, flavor='raw') if hasattr(_c, 'inet_ntop'): def inet_ntop(family, packed): "packed string -> human-readable string" - XXX if family == AF_INET: srcsize = sizeof(_c.in_addr) dstsize = _c.INET_ADDRSTRLEN @@ -1167,13 +1169,18 @@ raise RSocketError("unknown address family") if len(packed) != srcsize: raise ValueError("packed IP wrong length for inet_ntop") - srcbuf = create_string_buffer(srcsize) - srcbuf.raw = packed - dstbuf = create_string_buffer(dstsize) - res = _c.inet_ntop(family, cast(srcbuf, c_void_p), dstbuf, dstsize) - if res is None: - raise last_error() - return res + srcbuf = rffi.str2charp(packed) + try: + dstbuf = mallocbuf(dstsize) + try: + res = _c.inet_ntop(family, srcbuf, dstbuf, dstsize) + if not res: + raise last_error() + return rffi.charp2str(res) + finally: + lltype.free(dstbuf, flavor='raw') + finally: + lltype.free(srcbuf, flavor='raw') def setdefaulttimeout(timeout): if timeout < 0.0: Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 15:57:41 2007 @@ -276,6 +276,13 @@ def test_inet_ntoa(): assert inet_ntoa('\x01\x02\x03\x04') == '1.2.3.4' +def test_inet_pton(): + assert inet_pton(AF_INET, '1.2.3.5') == '\x01\x02\x03\x05' + py.test.raises(SocketError, inet_pton, AF_INET, '127.0.0.256') + +def test_inet_ntop(): + assert inet_ntop(AF_INET, '\x01\x02\x03\x05') == '1.2.3.5' + class TestTCP: PORT = 50007 HOST = 'localhost' From arigo at codespeak.net Thu Sep 20 16:03:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 16:03:56 +0200 (CEST) Subject: [pypy-svn] r46768 - in pypy/dist/pypy/rlib: . test Message-ID: <20070920140356.044638183@code0.codespeak.net> Author: arigo Date: Thu Sep 20 16:03:56 2007 New Revision: 46768 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: All tests pass on Linux. Still many XXXs left on Windows. Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 16:03:56 2007 @@ -509,14 +509,17 @@ if self.timeout <= 0.0 or self.fd < 0: # blocking I/O or no socket. return 0 - pollfd = _c.pollfd() - pollfd.fd = self.fd - if for_writing: - pollfd.events = _c.POLLOUT - else: - pollfd.events = _c.POLLIN - timeout = int(self.timeout * 1000.0 + 0.5) - n = _c.poll(byref(pollfd), 1, timeout) + pollfd = rffi.make(_c.pollfd) + try: + rffi.setintfield(pollfd, 'c_fd', self.fd) + if for_writing: + rffi.setintfield(pollfd, 'c_events', _c.POLLOUT) + else: + rffi.setintfield(pollfd, 'c_events', _c.POLLIN) + timeout = int(self.timeout * 1000.0 + 0.5) + n = _c.poll(pollfd, 1, timeout) + finally: + lltype.free(pollfd, flavor='raw') if n < 0: return -1 if n == 0: @@ -527,6 +530,7 @@ def _select(self, for_writing): """Returns 0 when reading/writing is possible, 1 when timing out and -1 on error.""" + XXX if self.timeout <= 0.0 or self.fd < 0: # blocking I/O or no socket. return 0 Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 16:03:56 2007 @@ -288,7 +288,6 @@ HOST = 'localhost' def setup_method(self, method): - py.test.skip("in-progress") self.serv = RSocket(AF_INET, SOCK_STREAM) self.serv.setsockopt_int(SOL_SOCKET, SO_REUSEADDR, 1) self.serv.bind(INETAddress(self.HOST, self.PORT)) From arigo at codespeak.net Thu Sep 20 16:46:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Sep 2007 16:46:50 +0200 (CEST) Subject: [pypy-svn] r46769 - in pypy/dist/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20070920144650.25FF980B4@code0.codespeak.net> Author: arigo Date: Thu Sep 20 16:46:49 2007 New Revision: 46769 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: gethostbyname_ex() test and fix. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Thu Sep 20 16:46:49 2007 @@ -236,7 +236,7 @@ ('h_aliases', rffi.CCHARPP), ('h_addrtype', rffi.INT), ('h_length', rffi.INT), - ('h_addr_list', rffi.VOIDP), + ('h_addr_list', rffi.CCHARPP), ]) Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Thu Sep 20 16:46:49 2007 @@ -981,37 +981,33 @@ def gethost_common(hostname, hostent, addr=None): if not hostent: raise HSocketError(hostname) - family = hostent.contents.h_addrtype + family = hostent.c_h_addrtype if addr is not None and addr.family != family: raise CSocketError(_c.EAFNOSUPPORT) - aliases = [] - h_aliases = hostent.contents.h_aliases + h_aliases = hostent.c_h_aliases if h_aliases: # h_aliases can be NULL, according to SF #1511317 - i = 0 - alias = h_aliases[0] - while alias is not None: - aliases.append(alias) - i += 1 - alias = h_aliases[i] + aliases = rffi.charpp2liststr(h_aliases) + else: + aliases = [] address_list = [] - h_addr_list = hostent.contents.h_addr_list + h_addr_list = hostent.c_h_addr_list i = 0 paddr = h_addr_list[0] while paddr: if family == AF_INET: - p = cast(paddr, POINTER(_c.in_addr)) - addr = INETAddress.from_in_addr(p.contents) + p = rffi.cast(lltype.Ptr(_c.in_addr), paddr) + addr = INETAddress.from_in_addr(p) elif AF_INET6 is not None and family == AF_INET6: - p = cast(paddr, POINTER(_c.in6_addr)) - addr = INET6Address.from_in6_addr(p.contents) + p = cast(lltype.Ptr(_c.in6_addr), paddr) + addr = INET6Address.from_in6_addr(p) else: raise RSocketError("unknown address family") address_list.append(addr) i += 1 paddr = h_addr_list[i] - return (hostent.contents.h_name, aliases, address_list) + return (rffi.charp2str(hostent.c_h_name), aliases, address_list) def gethostbyname_ex(name): # XXX use gethostbyname_r() if available, and/or use locks if not Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Thu Sep 20 16:46:49 2007 @@ -43,6 +43,19 @@ assert isinstance(a, INETAddress) assert a.get_host() == "127.0.0.1" +def test_gethostbyname_ex(): + name, aliases, address_list = gethostbyname_ex('localhost') + allnames = [name] + aliases + for n in allnames: + assert isinstance(n, str) + assert 'localhost' in allnames + for a in address_list: + if isinstance(a, INETAddress) and a.get_host() == "127.0.0.1": + break # ok + else: + py.test.fail("could not find the 127.0.0.1 IPv4 address in %r" + % (address_list,)) + def test_socketpair(): if sys.platform == "win32": py.test.skip('No socketpair on Windows') Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Sep 20 16:46:49 2007 @@ -321,6 +321,16 @@ i += 1 lltype.free(ref, flavor='raw') +def charpp2liststr(p): + """ char** NULL terminated -> list[str]. No freeing is done. + """ + result = [] + i = 0 + while p[i]: + result.append(charp2str(p[i])) + i += 1 + return result + cast = ll2ctypes.force_cast # a forced, no-checking cast From pypy-svn at codespeak.net Thu Sep 20 22:28:05 2007 From: pypy-svn at codespeak.net (Viagra.com Inc) Date: Thu, 20 Sep 2007 22:28:05 +0200 (CEST) Subject: [pypy-svn] Lovers package at discount price! Message-ID: <20070920112937.3382.qmail@host30-221-dynamic.11-79-r.retail.telecomitalia.it> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Fri Sep 21 11:21:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 11:21:07 +0200 (CEST) Subject: [pypy-svn] r46777 - in pypy/dist/pypy/rlib: . test Message-ID: <20070921092107.BFD89815A@code0.codespeak.net> Author: arigo Date: Fri Sep 21 11:21:06 2007 New Revision: 46777 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: Trying to do the right thing about the lifetime of the 'addr' of Address instances. See comments before the lock()/unlock() methods. It fixes an issue where lltype and ll2ctypes complain about a structures being used after they are freed, so I guess it also fixes a real issue in the generated C code. Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Fri Sep 21 11:21:06 2007 @@ -14,7 +14,7 @@ # It's unclear if makefile() and SSL support belong here or only as # app-level code for PyPy. -from pypy.rlib.objectmodel import instantiate +from pypy.rlib.objectmodel import instantiate, keepalive_until_here from pypy.rlib import _rsocket_rffi as _c from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, rffi @@ -58,32 +58,58 @@ return A # default uninitialized value: NULL ptr - addr = lltype.nullptr(_c.sockaddr_ptr.TO) + addr_p = lltype.nullptr(_c.sockaddr_ptr.TO) def __init__(self, addr, addrlen): - self.addr = addr + self.addr_p = addr self.addrlen = addrlen def __del__(self): - addr = self.addr + addr = self.addr_p if addr: lltype.free(addr, flavor='raw') def setdata(self, addr, addrlen): # initialize self.addr and self.addrlen. 'addr' can be a different # pointer type than exactly sockaddr_ptr, and we cast it for you. - assert not self.addr - self.addr = rffi.cast(_c.sockaddr_ptr, addr) + assert not self.addr_p + self.addr_p = rffi.cast(_c.sockaddr_ptr, addr) self.addrlen = addrlen setdata._annspecialcase_ = 'specialize:ll' + # the following slightly strange interface is needed to manipulate + # what self.addr_p points to in a safe way. The problem is that + # after inlining we might end up with operations that looks like: + # addr = self.addr_p + # + # read from addr + # To prevent this we have to insert a keepalive after the last + # use of 'addr'. The interface to do that is called lock()/unlock() + # because it strongly reminds callers not to forget unlock(). + # + def lock(self, TYPE=_c.sockaddr): + """Return self.addr_p, cast as a pointer to TYPE. Must call unlock()! + """ + if not (self.minlen <= self.addrlen <= self.maxlen): + raise RSocketError("invalid address") + return rffi.cast(lltype.Ptr(TYPE), self.addr_p) + lock._annspecialcase_ = 'specialize:ll' + + def unlock(self): + """To call after we're done with the pointer returned by lock(). + Note that locking and unlocking costs nothing at run-time. + """ + keepalive_until_here(self) + def as_object(self, space): """Convert the address to an app-level object.""" # If we don't know the address family, don't raise an # exception -- return it as a tuple. - family = rffi.cast(lltype.Signed, self.addr.c_sa_family) + addr = self.lock() + family = rffi.cast(lltype.Signed, addr.c_sa_family) datalen = self.addrlen - offsetof(_c.sockaddr, 'c_sa_data') - rawdata = ''.join([self.addr.c_sa_data[i] for i in range(datalen)]) + rawdata = ''.join([addr.c_sa_data[i] for i in range(datalen)]) + self.unlock() return space.newtuple([space.wrap(family), space.wrap(rawdata)]) @@ -160,17 +186,13 @@ class INETAddress(IPAddress): family = AF_INET struct = _c.sockaddr_in - maxlen = sizeof(struct) + maxlen = minlen = sizeof(struct) def __init__(self, host, port): makeipaddr(host, self) - a = self.as_sockaddr_in() + a = self.lock(_c.sockaddr_in) a.c_sin_port = htons(port) - - def as_sockaddr_in(self): - if self.addrlen != INETAddress.maxlen: - raise RSocketError("invalid address") - return rffi.cast(lltype.Ptr(_c.sockaddr_in), self.addr) + self.unlock() def __repr__(self): try: @@ -179,8 +201,10 @@ return '' def get_port(self): - a = self.as_sockaddr_in() - return ntohs(a.c_sin_port) + a = self.lock(_c.sockaddr_in) + port = ntohs(a.c_sin_port) + self.unlock() + return port def eq(self, other): # __eq__() is not called by RPython :-/ return (isinstance(other, INETAddress) and @@ -206,8 +230,9 @@ # XXX a bit of code duplication _, w_port = space.unpackiterable(w_address, 2) port = space.int_w(w_port) - a = self.as_sockaddr_in() + a = self.lock(_c.sockaddr_in) a.c_sin_port = htons(port) + self.unlock() def from_in_addr(in_addr): result = instantiate(INETAddress) @@ -221,8 +246,9 @@ return result from_in_addr = staticmethod(from_in_addr) - def extract_in_addr(self): - p = rffi.cast(rffi.VOIDP, self.as_sockaddr_in().sin_addr) + def lock_in_addr(self): + a = self.lock(_c.sockaddr_in) + p = rffi.cast(rffi.VOIDP, a.c_sin_addr) return p, sizeof(_c.in_addr) # ____________________________________________________________ @@ -230,19 +256,15 @@ class INET6Address(IPAddress): family = AF_INET6 struct = _c.sockaddr_in6 - maxlen = sizeof(struct) + maxlen = minlen = sizeof(struct) def __init__(self, host, port, flowinfo=0, scope_id=0): makeipaddr(host, self) - a = self.as_sockaddr_in6() + a = self.lock(_c.sockaddr_in6) a.c_sin6_port = htons(port) a.c_sin6_flowinfo = flowinfo a.c_sin6_scope_id = scope_id - - def as_sockaddr_in6(self): - if self.addrlen != INET6Address.maxlen: - raise RSocketError("invalid address") - return rffi.cast(lltype.Ptr(_c.sockaddr_in6), self.addr) + self.unlock() def __repr__(self): try: @@ -254,16 +276,22 @@ return '' def get_port(self): - a = self.as_sockaddr_in6() - return ntohs(a.c_sin6_port) + a = self.lock(_c.sockaddr_in6) + port = ntohs(a.c_sin6_port) + self.unlock() + return port def get_flowinfo(self): - a = self.as_sockaddr_in6() - return a.c_sin6_flowinfo + a = self.lock(_c.sockaddr_in6) + flowinfo = a.c_sin6_flowinfo + self.unlock() + return flowinfo def get_scope_id(self): - a = self.as_sockaddr_in6() - return a.c_sin6_scope_id + a = self.lock(_c.sockaddr_in6) + scope_id = a.c_sin6_scope_id + self.unlock() + return scope_id def eq(self, other): # __eq__() is not called by RPython :-/ return (isinstance(other, INET6Address) and @@ -303,10 +331,11 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.int_w(pieces_w[3]) else: scope_id = 0 - a = self.as_sockaddr_in6() + a = self.lock(_c.sockaddr_in6) a.c_sin6_port = htons(port) a.c_sin6_flowinfo = flowinfo a.c_sin6_scope_id = scope_id + self.unlock() def from_in6_addr(in6_addr): result = instantiate(INET6Address) @@ -319,8 +348,9 @@ return result from_in6_addr = staticmethod(from_in6_addr) - def extract_in_addr(self): - p = rffi.cast(rffi.VOIDP, self.as_sockaddr_in6().sin6_addr) + def lock_in_addr(self): + a = self.lock(_c.sockaddr_in6) + p = rffi.cast(rffi.VOIDP, a.c_sin6_addr) return p, sizeof(_c.in6_addr) # ____________________________________________________________ @@ -329,6 +359,7 @@ class UNIXAddress(Address): family = AF_UNIX struct = _c.sockaddr_un + minlen = offsetof(_c.sockaddr_un, 'c_sun_path') + 1 maxlen = sizeof(struct) def __init__(self, path): @@ -348,11 +379,6 @@ for i in range(len(path)): sun.c_sun_path[i] = path[i] - def as_sockaddr_un(self): - if self.addrlen <= offsetof(_c.sockaddr_un, 'c_sun_path'): - raise RSocketError("invalid address") - return rffi.cast(lltype.Ptr(_c.sockaddr_un), self.addr) - def __repr__(self): try: return '' % (self.get_path(),) @@ -360,7 +386,7 @@ return '' def get_path(self): - a = self.as_sockaddr_un() + a = self.lock(_c.sockaddr_un) maxlength = self.addrlen - offsetof(_c.sockaddr_un, 'c_sun_path') if _c.linux and a.c_sun_path[0] == '\x00': # Linux abstract namespace @@ -370,7 +396,9 @@ length = 0 while length < maxlength and a.c_sun_path[length] != '\x00': length += 1 - return ''.join([a.c_sun_path[i] for i in range(length)]) + result = ''.join([a.c_sun_path[i] for i in range(length)]) + self.unlock() + return result def eq(self, other): # __eq__() is not called by RPython :-/ return (isinstance(other, UNIXAddress) and @@ -387,7 +415,7 @@ class NETLINKAddress(Address): family = AF_NETLINK struct = _c.sockaddr_nl - maxlen = sizeof(struct) + maxlen = minlen = sizeof(struct) def __init__(self, pid, groups): addr = rffi.make(_c.sockaddr_nl) @@ -396,16 +424,17 @@ rffi.setintfield(addr, 'c_nl_pid', pid) rffi.setintfield(addr, 'c_nl_groups', groups) - def as_sockaddr_nl(self): - if self.addrlen != NETLINKAddress.maxlen: - raise RSocketError("invalid address") - return rffi.cast(lltype.Ptr(_c.sockaddr_nl), self.addr) - def get_pid(self): - return self.as_sockaddr_nl().c_nl_pid + a = self.lock(_c.sockaddr_nl) + pid = a.c_nl_pid + self.unlock() + return pid def get_groups(self): - return self.as_sockaddr_nl().c_nl_groups + a = self.lock(_c.sockaddr_nl) + groups = a.c_nl_groups + self.unlock() + return groups def __repr__(self): return '' % (self.get_pid(), self.get_groups()) @@ -562,11 +591,14 @@ def addr_from_object(self, space, w_address): return af_get(self.family).from_object(space, w_address) + # build a null address object, ready to be used as output argument to + # C functions that return an address. It must be unlock()ed after you + # are done using addr_p. def _addrbuf(self): addr, maxlen = make_null_address(self.family) addrlen_p = lltype.malloc(_c.socklen_t_ptr.TO, flavor='raw') addrlen_p[0] = rffi.cast(_c.socklen_t, maxlen) - return addr, addrlen_p + return addr, addr.addr_p, addrlen_p def accept(self, SocketClass=None): """Wait for an incoming connection. @@ -575,12 +607,13 @@ SocketClass = RSocket if self._select(False) == 1: raise SocketTimeout - address, addrlen_p = self._addrbuf() + address, addr_p, addrlen_p = self._addrbuf() try: - newfd = _c.socketaccept(self.fd, address.addr, addrlen_p) + newfd = _c.socketaccept(self.fd, addr_p, addrlen_p) addrlen = addrlen_p[0] finally: lltype.free(addrlen_p, flavor='raw') + address.unlock() if _c.invalid_socket(newfd): raise self.error_handler() address.addrlen = addrlen @@ -590,7 +623,9 @@ def bind(self, address): """Bind the socket to a local address.""" - res = _c.socketbind(self.fd, address.addr, address.addrlen) + addr = address.lock() + res = _c.socketbind(self.fd, addr, address.addrlen) + address.unlock() if res < 0: raise self.error_handler() @@ -605,14 +640,17 @@ def connect(self, address): """Connect the socket to a remote address.""" - res = _c.socketconnect(self.fd, address.addr, address.addrlen) + addr = address.lock() + res = _c.socketconnect(self.fd, addr, address.addrlen) + address.unlock() if self.timeout > 0.0: errno = _c.geterrno() if res < 0 and errno == _c.EINPROGRESS: timeout = self._select(True) if timeout == 0: - res = _c.socketconnect(self.fd, address.addr, - address.addrlen) + addr = address.lock() + res = _c.socketconnect(self.fd, addr, address.addrlen) + address.unlock() elif timeout == -1: raise self.error_handler() else: @@ -624,14 +662,17 @@ def connect_ex(self, address): """This is like connect(address), but returns an error code (the errno value) instead of raising an exception when an error occurs.""" - res = _c.socketconnect(self.fd, address.addr, address.addrlen) + addr = address.lock() + res = _c.socketconnect(self.fd, addr, address.addrlen) + address.unlock() if self.timeout > 0.0: errno = _c.geterrno() if res < 0 and errno == _c.EINPROGRESS: timeout = self._select(True) if timeout == 0: - res = _c.socketconnect(self.fd, address.addr, - address.addrlen) + addr = address.lock() + res = _c.socketconnect(self.fd, addr, address.addrlen) + address.unlock() elif timeout == -1: return _c.geterrno() else: @@ -659,12 +700,13 @@ def getpeername(self): """Return the address of the remote endpoint.""" - address, addrlen_p = self._addrbuf() + address, addr_p, addrlen_p = self._addrbuf() try: - res = _c.socketgetpeername(self.fd, address.addr, addrlen_p) + res = _c.socketgetpeername(self.fd, addr_p, addrlen_p) addrlen = addrlen_p[0] finally: lltype.free(addrlen_p, flavor='raw') + address.unlock() if res < 0: raise self.error_handler() address.addrlen = addrlen @@ -672,12 +714,13 @@ def getsockname(self): """Return the address of the local endpoint.""" - address, addrlen_p = self._addrbuf() + address, addr_p, addrlen_p = self._addrbuf() try: - res = _c.socketgetsockname(self.fd, address.addr, addrlen_p) + res = _c.socketgetsockname(self.fd, addr_p, addrlen_p) addrlen = addrlen_p[0] finally: lltype.free(addrlen_p, flavor='raw') + address.unlock() if res < 0: raise self.error_handler() address.addrlen = addrlen @@ -765,13 +808,14 @@ elif timeout == 0: buf = mallocbuf(buffersize) try: - address, addrlen_p = self._addrbuf() + address, addr_p, addrlen_p = self._addrbuf() try: read_bytes = _c.recvfrom(self.fd, buf, buffersize, flags, - address.addr, addrlen_p) + addr_p, addrlen_p) addrlen = addrlen_p[0] finally: lltype.free(addrlen_p, flavor='raw') + address.unlock() if read_bytes >= 0: if addrlen: address.addrlen = addrlen @@ -814,8 +858,10 @@ if timeout == 1: raise SocketTimeout elif timeout == 0: + addr = address.lock() res = _c.sendto(self.fd, data, len(data), flags, - address.addr, address.addrlen) + addr, address.addrlen) + address.unlock() if res < 0: raise self.error_handler() return res @@ -1018,8 +1064,11 @@ def gethostbyaddr(ip): # XXX use gethostbyaddr_r() if available, and/or use locks if not addr = makeipaddr(ip) - p, size = addr.extract_in_addr() - hostent =_c.gethostbyaddr(p, size, addr.family) + p, size = addr.lock_in_addr() + try: + hostent = _c.gethostbyaddr(p, size, addr.family) + finally: + addr.unlock() return gethost_common(ip, hostent, addr) def getaddrinfo(host, port_or_service, @@ -1078,14 +1127,16 @@ raise RSocketError("protocol not found") return protoent.contents.p_proto -def getnameinfo(addr, flags): +def getnameinfo(address, flags): host = lltype.malloc(rffi.CCHARP.TO, NI_MAXHOST, flavor='raw') try: serv = lltype.malloc(rffi.CCHARP.TO, NI_MAXSERV, flavor='raw') try: - error =_c.getnameinfo(addr.addr, addr.addrlen, + addr = address.lock() + error =_c.getnameinfo(addr, address.addrlen, host, NI_MAXHOST, serv, NI_MAXSERV, flags) + address.unlock() if error: raise GAIError(error) return rffi.charp2str(host), rffi.charp2str(serv) Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Fri Sep 21 11:21:06 2007 @@ -56,6 +56,19 @@ py.test.fail("could not find the 127.0.0.1 IPv4 address in %r" % (address_list,)) +def test_gethostbyaddr(): + name, aliases, address_list = gethostbyaddr('127.0.0.1') + allnames = [name] + aliases + for n in allnames: + assert isinstance(n, str) + assert 'localhost' in allnames + for a in address_list: + if isinstance(a, INETAddress) and a.get_host() == "127.0.0.1": + break # ok + else: + py.test.fail("could not find the 127.0.0.1 IPv4 address in %r" + % (address_list,)) + def test_socketpair(): if sys.platform == "win32": py.test.skip('No socketpair on Windows') From arigo at codespeak.net Fri Sep 21 11:27:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 11:27:11 +0200 (CEST) Subject: [pypy-svn] r46778 - in pypy/dist/pypy/rlib: . test Message-ID: <20070921092711.6E868815C@code0.codespeak.net> Author: arigo Date: Fri Sep 21 11:27:11 2007 New Revision: 46778 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: Tests and fixes. Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Fri Sep 21 11:27:11 2007 @@ -1113,19 +1113,19 @@ servent = _c.getservbyname(name, proto) if not servent: raise RSocketError("service/proto not found") - return _c.ntohs(servent.contents.s_port) + return _c.ntohs(servent.c_s_port) def getservbyport(port, proto=None): servent = _c.getservbyport(htons(port), proto) if not servent: raise RSocketError("port/proto not found") - return servent.contents.s_name + return rffi.charp2str(servent.c_s_name) def getprotobyname(name): protoent = _c.getprotobyname(name) if not protoent: raise RSocketError("protocol not found") - return protoent.contents.p_proto + return protoent.c_p_proto def getnameinfo(address, flags): host = lltype.malloc(rffi.CCHARP.TO, NI_MAXHOST, flavor='raw') Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Fri Sep 21 11:27:11 2007 @@ -69,6 +69,18 @@ py.test.fail("could not find the 127.0.0.1 IPv4 address in %r" % (address_list,)) +def test_getservbyname(): + assert getservbyname('http') == 80 + assert getservbyname('http', 'tcp') == 80 + +def test_getservbyport(): + assert getservbyport(80) == 'http' + assert getservbyport(80, 'tcp') == 'http' + +def test_getprotobyname(): + assert getprotobyname('tcp') == IPPROTO_TCP + assert getprotobyname('udp') == IPPROTO_UDP + def test_socketpair(): if sys.platform == "win32": py.test.skip('No socketpair on Windows') From arigo at codespeak.net Fri Sep 21 11:38:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 11:38:32 +0200 (CEST) Subject: [pypy-svn] r46779 - pypy/dist/pypy/objspace/fake Message-ID: <20070921093832.695768171@code0.codespeak.net> Author: arigo Date: Fri Sep 21 11:38:32 2007 New Revision: 46779 Modified: pypy/dist/pypy/objspace/fake/checkmodule.py Log: Whack a bit at objspace.fake.checkmodule to let it process module/_socket. Modified: pypy/dist/pypy/objspace/fake/checkmodule.py ============================================================================== --- pypy/dist/pypy/objspace/fake/checkmodule.py (original) +++ pypy/dist/pypy/objspace/fake/checkmodule.py Fri Sep 21 11:38:32 2007 @@ -1,3 +1,4 @@ +import re from copy import copy from pypy.tool.error import debug from pypy.interpreter.argument import AbstractArguments @@ -12,19 +13,24 @@ return mod def find_gateways(modname, basepath, module): + identifier = r'[a-zA-Z0-9][a-zA-Z0-9_]*' + r_simplename = re.compile(r'(%s)[.](%s)$' % (identifier, identifier)) res = [] for name in module.interpleveldefs.values(): - submod_name, obj_name = name.split('.') - submod_name = '%s.%s.%s' % (basepath, modname, submod_name) - submod = my_import(submod_name) - obj = getattr(submod, obj_name) - res += find_gw_in_obj(obj) + match = r_simplename.match(name) + if match: + submod_name, obj_name = match.groups() + submod_name = '%s.%s.%s' % (basepath, modname, submod_name) + submod = my_import(submod_name) + obj = getattr(submod, obj_name) + res += find_gw_in_obj(obj) return res def find_gw_in_obj(obj): if hasattr(obj, 'typedef'): typedef = obj.typedef - return typedef.rawdict.values() # XXX: check they are interp2app + return [gw for gw in typedef.rawdict.values() + if isinstance(gw, interp2app)] elif hasattr(obj, 'func_code'): return [interp2app(obj)] else: From arigo at codespeak.net Fri Sep 21 11:52:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 11:52:21 +0200 (CEST) Subject: [pypy-svn] r46780 - in pypy/dist/pypy: rlib rpython rpython/lltypesystem Message-ID: <20070921095221.6C1368155@code0.codespeak.net> Author: arigo Date: Fri Sep 21 11:52:20 2007 New Revision: 46780 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/rbuiltin.py Log: First round of translation fixes. 'checkmodule' is great :-) Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Fri Sep 21 11:52:20 2007 @@ -581,7 +581,7 @@ assert len(WIN32_ERROR_MESSAGES) == 53 # detect duplicates - def socket_strerror(errno): + def socket_strerror_str(errno): return WIN32_ERROR_MESSAGES.get(errno, "winsock error %d" % errno) else: - socket_strerror = os.strerror + socket_strerror_str = os.strerror Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Fri Sep 21 11:52:20 2007 @@ -231,7 +231,7 @@ _, w_port = space.unpackiterable(w_address, 2) port = space.int_w(w_port) a = self.lock(_c.sockaddr_in) - a.c_sin_port = htons(port) + rffi.setintfield(a, 'c_sin_port', htons(port)) self.unlock() def from_in_addr(in_addr): @@ -313,9 +313,9 @@ "to 4, not %d" % len(pieces_w)) host = space.str_w(pieces_w[0]) port = space.int_w(pieces_w[1]) - if len(pieces_w) > 2: flowinfo = space.int_w(pieces_w[2]) + if len(pieces_w) > 2: flowinfo = space.uint_w(pieces_w[2]) else: flowinfo = 0 - if len(pieces_w) > 3: scope_id = space.int_w(pieces_w[3]) + if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 return INET6Address(host, port, flowinfo, scope_id) from_object = staticmethod(from_object) @@ -327,14 +327,14 @@ raise RSocketError("AF_INET6 address must be a tuple of length 2 " "to 4, not %d" % len(pieces_w)) port = space.int_w(pieces_w[1]) - if len(pieces_w) > 2: flowinfo = space.int_w(pieces_w[2]) + if len(pieces_w) > 2: flowinfo = space.uint_w(pieces_w[2]) else: flowinfo = 0 - if len(pieces_w) > 3: scope_id = space.int_w(pieces_w[3]) + if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 a = self.lock(_c.sockaddr_in6) - a.c_sin6_port = htons(port) - a.c_sin6_flowinfo = flowinfo - a.c_sin6_scope_id = scope_id + rffi.setintfield(a, 'c_sin6_port', htons(port)) + rffi.setintfield(a, 'c_sin6_flowinfo', flowinfo) + rffi.setintfield(a, 'c_sin6_scope_id', scope_id) self.unlock() def from_in6_addr(in6_addr): @@ -458,7 +458,7 @@ af_get = familyclass def make_address(addrptr, addrlen, result=None): - family = addrptr.c_sa_family + family = rffi.cast(lltype.Signed, addrptr.c_sa_family) if result is None: result = instantiate(familyclass(family)) elif result.family != family: @@ -940,7 +940,7 @@ class CSocketError(SocketErrorWithErrno): def get_msg(self): - return _c.socket_strerror(self.errno) + return _c.socket_strerror_str(self.errno) def last_error(): return CSocketError(_c.geterrno()) @@ -948,7 +948,7 @@ class GAIError(SocketErrorWithErrno): applevelerrcls = 'gaierror' def get_msg(self): - return _c.gai_strerror(self.errno) + return rffi.charp2str(_c.gai_strerror(self.errno)) class HSocketError(SocketError): applevelerrcls = 'herror' @@ -1046,7 +1046,7 @@ p = rffi.cast(lltype.Ptr(_c.in_addr), paddr) addr = INETAddress.from_in_addr(p) elif AF_INET6 is not None and family == AF_INET6: - p = cast(lltype.Ptr(_c.in6_addr), paddr) + p = rffi.cast(lltype.Ptr(_c.in6_addr), paddr) addr = INET6Address.from_in6_addr(p) else: raise RSocketError("unknown address family") @@ -1098,9 +1098,9 @@ canonname = rffi.charp2str(info.c_ai_canonname) else: canonname = "" - result.append((info.c_ai_family, - info.c_ai_socktype, - info.c_ai_protocol, + result.append((rffi.cast(lltype.Signed, info.c_ai_family), + rffi.cast(lltype.Signed, info.c_ai_socktype), + rffi.cast(lltype.Signed, info.c_ai_protocol), canonname, addr)) info = info.c_ai_next Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Fri Sep 21 11:52:20 2007 @@ -369,7 +369,7 @@ if tp is lltype.Signed: return ULONG._type.BITS/8 return tp._type.BITS/8 -_annspecialcase_ = 'specialize:memo' +sizeof._annspecialcase_ = 'specialize:memo' def offsetof(STRUCT, fieldname): """Similar to llmemory.offsetof() but tries hard to return a integer @@ -385,7 +385,7 @@ # a symbolic result as a fallback from pypy.rpython.lltypesystem import llmemory return llmemory.offsetof(STRUCT, fieldname) -_annspecialcase_ = 'specialize:memo' +offsetof._annspecialcase_ = 'specialize:memo' # ********************** some helpers ******************* Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Fri Sep 21 11:52:20 2007 @@ -363,6 +363,7 @@ hop.genop('free', vlist) def rtype_const_result(hop): + hop.exception_cannot_occur() return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const) def rtype_cast_pointer(hop): From arigo at codespeak.net Fri Sep 21 11:56:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 11:56:42 +0200 (CEST) Subject: [pypy-svn] r46781 - pypy/dist/pypy/translator Message-ID: <20070921095642.B3B5780A8@code0.codespeak.net> Author: arigo Date: Fri Sep 21 11:56:42 2007 New Revision: 46781 Modified: pypy/dist/pypy/translator/exceptiontransform.py Log: Teach exceptiontransform about the types like rffi.USHORT. Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Fri Sep 21 11:56:42 2007 @@ -4,7 +4,7 @@ from pypy.translator.backendopt import canraise, inline, support, removenoops from pypy.objspace.flow.model import Block, Constant, Variable, Link, \ c_last_exception, SpaceOperation, checkgraph, FunctionGraph -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem import lloperation from pypy.rpython.memory.lladdress import NULL @@ -26,6 +26,10 @@ llmemory.Address: NULL, lltype.Void: None} +for TYPE in rffi.NUMBER_TYPES: + PrimitiveErrorValue[TYPE] = rffi.cast(TYPE, -1) +del TYPE + def error_value(T): if isinstance(T, lltype.Primitive): return PrimitiveErrorValue[T] From arigo at codespeak.net Fri Sep 21 12:14:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 12:14:19 +0200 (CEST) Subject: [pypy-svn] r46782 - pypy/dist/pypy/rlib Message-ID: <20070921101419.6B7DF8167@code0.codespeak.net> Author: arigo Date: Fri Sep 21 12:14:18 2007 New Revision: 46782 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py Log: Add the includes needed for the external functions. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Fri Sep 21 12:14:18 2007 @@ -33,6 +33,7 @@ COND_HEADER = ''.join(['#ifdef %s\n#include <%s>\n#endif\n' % cond_include for cond_include in cond_includes]) if _MS_WINDOWS: + includes = ('WinSock2.h', 'WS2tcpip.h') HEADER = '\n'.join([ '#include ', '#include ', @@ -376,8 +377,15 @@ if MS_WINDOWS: fd_set = cConfig.fd_set -#c_int_size = sizeof(rffi.INT) -external = rffi.llexternal + +if _POSIX: + includes = list(includes) + for _name, _header in cond_includes: + if getattr(cConfig, _name) is not None: + includes.append(_header) + +def external(name, args, result): + return rffi.llexternal(name, args, result, includes=includes) if _POSIX: dup = external('dup', [socketfd_type], socketfd_type) From arigo at codespeak.net Fri Sep 21 12:14:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 12:14:45 +0200 (CEST) Subject: [pypy-svn] r46783 - in pypy/dist/pypy/rpython: lltypesystem tool Message-ID: <20070921101445.B49548170@code0.codespeak.net> Author: arigo Date: Fri Sep 21 12:14:45 2007 New Revision: 46783 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Don't copy the _pad0, _pad1... fields, which don't really exist in the C headers. Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Fri Sep 21 12:14:45 2007 @@ -434,7 +434,9 @@ assert PDST == PSRC if isinstance(PDST.TO, lltype.Struct): STRUCT = PDST.TO - fields = [(name, STRUCT._flds[name]) for name in STRUCT._names] + padding = STRUCT._hints.get('padding', ()) + fields = [(name, STRUCT._flds[name]) for name in STRUCT._names + if name not in padding] unrollfields = unrolling_iterable(fields) def copyfn(pdst, psrc): Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Fri Sep 21 12:14:45 2007 @@ -231,10 +231,13 @@ layout_addfield(layout, offset, fieldtype, fieldname) n = 0 + padfields = [] for i, cell in enumerate(layout): if cell is not None: continue - layout_addfield(layout, i, rffi.UCHAR, '_pad%d' % (n,)) + name = '_pad%d' % (n,) + layout_addfield(layout, i, rffi.UCHAR, name) + padfields.append('c_' + name) n += 1 # build the lltype Structure @@ -253,7 +256,8 @@ name = name[7:] kwds = {'hints':{'align':info['align'], 'size':info['size'], - 'fieldoffsets':tuple(fieldoffsets)}} + 'fieldoffsets':tuple(fieldoffsets), + 'padding':tuple(padfields)}} return rffi.CStruct(name, *fields, **kwds) class SimpleType(CConfigEntry): From arigo at codespeak.net Fri Sep 21 12:21:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 12:21:49 +0200 (CEST) Subject: [pypy-svn] r46784 - pypy/dist/pypy/translator/c/src Message-ID: <20070921102149.5153C8170@code0.codespeak.net> Author: arigo Date: Fri Sep 21 12:21:49 2007 New Revision: 46784 Modified: pypy/dist/pypy/translator/c/src/support.h Log: Oups! Modified: pypy/dist/pypy/translator/c/src/support.h ============================================================================== --- pypy/dist/pypy/translator/c/src/support.h (original) +++ pypy/dist/pypy/translator/c/src/support.h Fri Sep 21 12:21:49 2007 @@ -78,7 +78,7 @@ # define RPyNLenItem(array, index) \ ((RPyCHECK((array) && (index) >= 0), (array))->items[index]) # define RPyBareItem(array, index) \ - ((RPyCHECK((array) && (index) >= 0), (array)[index]) + ((RPyCHECK((array) && (index) >= 0), (array))[index]) int RPyAbort(void); #ifndef PYPY_NOT_MAIN_FILE From arigo at codespeak.net Fri Sep 21 12:26:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 12:26:28 +0200 (CEST) Subject: [pypy-svn] r46785 - in pypy/dist/pypy/rlib: . test Message-ID: <20070921102628.4103B8170@code0.codespeak.net> Author: arigo Date: Fri Sep 21 12:26:27 2007 New Revision: 46785 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Log: Test and fix. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Fri Sep 21 12:26:27 2007 @@ -452,7 +452,7 @@ ssize_t) sendto = external('sendto', [socketfd_type, rffi.VOIDP, size_t, rffi.INT, sockaddr_ptr, socklen_t], ssize_t) -shutdown = external('shutdown', [socketfd_type, rffi.INT], rffi.INT) +socketshutdown = external('shutdown', [socketfd_type, rffi.INT], rffi.INT) gethostname = external('gethostname', [rffi.CCHARP, rffi.INT], rffi.INT) gethostbyname = external('gethostbyname', [rffi.CCHARP], lltype.Ptr(cConfig.hostent)) @@ -472,8 +472,6 @@ ioctlsocket = socketdll.ioctlsocket ioctlsocket.argtypes = [socketfd_type, rffi.LONG, POINTER(c_ulong)] ioctlsocket.restype = c_int - -shutdown = external('shutdown', [rffi.INT, rffi.INT], rffi.INT) if _POSIX: poll = external('poll', [lltype.Ptr(pollfd), nfds_t, rffi.INT], rffi.INT) Modified: pypy/dist/pypy/rlib/test/test_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Fri Sep 21 12:26:27 2007 @@ -136,6 +136,7 @@ buf += data assert buf == 'x'*50000 print 'data received ok' + s1.shutdown(SHUT_RDWR) s1.close() s2.close() From arigo at codespeak.net Fri Sep 21 12:29:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 12:29:43 +0200 (CEST) Subject: [pypy-svn] r46786 - pypy/dist/pypy/translator/c/test Message-ID: <20070921102943.D95B38170@code0.codespeak.net> Author: arigo Date: Fri Sep 21 12:29:43 2007 New Revision: 46786 Modified: pypy/dist/pypy/translator/c/test/test_typed.py Log: float ** -> math.pow(). Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Fri Sep 21 12:29:43 2007 @@ -1,5 +1,6 @@ import autopath import sys +import math import py from py.test import raises @@ -179,7 +180,7 @@ def test_float_ops(self): def f(x): - return abs((-x) ** 3 + 1) + return abs(math.pow(-x, 3) + 1) fn = self.getcompiled(f, [float]) assert fn(-4.5) == 92.125 assert fn(4.5) == 90.125 @@ -507,7 +508,7 @@ def func(x, y): z = x + y / 2.1 * x z = math.fmod(z, 60.0) - z = pow(z, 2) + z = math.pow(z, 2) z = -z return int(z) From cfbolz at codespeak.net Fri Sep 21 13:08:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 21 Sep 2007 13:08:10 +0200 (CEST) Subject: [pypy-svn] r46787 - pypy/dist/pypy/lang/prolog/interpreter Message-ID: <20070921110810.1AA168154@code0.codespeak.net> Author: cfbolz Date: Fri Sep 21 13:08:07 2007 New Revision: 46787 Modified: pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py Log: remove use of float ** Modified: pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py Fri Sep 21 13:08:07 2007 @@ -109,7 +109,7 @@ ("-", ["expr", "expr"], "v0 - v1", True, True), ("*", ["expr", "expr"], "v0 * v1", True, True), ("//", ["int", "int"], "v0 / v1", True, False), - ("**", ["expr", "expr"], "float(v0) ** float(v1)", True, False), + ("**", ["expr", "expr"], "math.pow(float(v0), float(v1))", True, False), (">>", ["int", "int"], "v0 >> v1", False, False), ("<<", ["int", "int"], "intmask(v0 << v1)", False, False), From arigo at codespeak.net Fri Sep 21 13:13:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 13:13:58 +0200 (CEST) Subject: [pypy-svn] r46788 - in pypy/dist/pypy/translator/c: . test Message-ID: <20070921111358.0E34F817D@code0.codespeak.net> Author: arigo Date: Fri Sep 21 13:13:57 2007 New Revision: 46788 Modified: pypy/dist/pypy/translator/c/funcgen.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/test/test_lltyped.py Log: More fixes for barebone arrays. Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Fri Sep 21 13:13:57 2007 @@ -2,7 +2,7 @@ from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring from pypy.translator.c.support import cdecl, ErrorValue from pypy.translator.c.support import llvalue_from_constant, gen_assignments -from pypy.translator.c.support import c_string_constant +from pypy.translator.c.support import c_string_constant, barebonearray from pypy.objspace.flow.model import Variable, Constant, Block from pypy.objspace.flow.model import c_last_exception, copygraph from pypy.rpython.lltypesystem.lltype import Ptr, PyObject, Void, Bool, Signed @@ -546,7 +546,7 @@ def OP_DIRECT_ARRAYITEMS(self, op): ARRAY = self.lltypemap(op.args[0]).TO items = self.expr(op.args[0]) - if not isinstance(ARRAY, FixedSizeArray): + if not isinstance(ARRAY, FixedSizeArray) and not barebonearray(ARRAY): items += '->items' return '%s = %s;' % (self.expr(op.result), items) Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Fri Sep 21 13:13:57 2007 @@ -253,7 +253,10 @@ return '%s %s *@' % (self.typetag, self.name) def access_expr(self, baseexpr, index): - return '%s.items[%d]' % (baseexpr, index) + if self.barebone: + return '%s[%d]' % (baseexpr, index) + else: + return '%s.items[%d]' % (baseexpr, index) def ptr_access_expr(self, baseexpr, index): assert 0 <= index <= sys.maxint, "invalid constant index %r" % (index,) Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Fri Sep 21 13:13:57 2007 @@ -130,7 +130,10 @@ def test_direct_arrayitems(self): for a in [malloc(GcArray(Signed), 5), - malloc(FixedSizeArray(Signed, 5), immortal=True)]: + malloc(FixedSizeArray(Signed, 5), immortal=True), + malloc(Array(Signed, hints={'nolength': True}), 5, + immortal=True), + ]: a[0] = 0 a[1] = 10 a[2] = 20 @@ -388,6 +391,31 @@ res = fn(100) assert res == 3050 + def test_structarray_nolength(self): + S = Struct('S', ('x', Signed)) + A = Array(S, hints={'nolength': True}) + a1 = malloc(A, 3, immortal=True) + a1[0].x = 30 + a1[1].x = 300 + a1[2].x = 3000 + a1dummy = malloc(A, 2, immortal=True) + + def f(n): + if n & 1: + src = a1dummy + else: + src = a1 + a2 = malloc(A, n, flavor='raw') + for i in range(n): + a2[i].x = src[i % 3].x + i + res = a2[n // 2].x + free(a2, flavor='raw') + return res + + fn = self.getcompiled(f, [int]) + res = fn(100) + assert res == 3050 + def test_arithmetic_cornercases(self): import operator, sys from pypy.rlib.unroll import unrolling_iterable From cfbolz at codespeak.net Fri Sep 21 13:28:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 21 Sep 2007 13:28:23 +0200 (CEST) Subject: [pypy-svn] r46789 - in pypy/dist/pypy/translator/js: . test Message-ID: <20070921112823.8D6768168@code0.codespeak.net> Author: cfbolz Date: Fri Sep 21 13:28:23 2007 New Revision: 46789 Modified: pypy/dist/pypy/translator/js/jsbuiltin.py pypy/dist/pypy/translator/js/test/test_typed.py Log: float ** -> math.pow. Implement math.pow in the js backend. Modified: pypy/dist/pypy/translator/js/jsbuiltin.py ============================================================================== --- pypy/dist/pypy/translator/js/jsbuiltin.py (original) +++ pypy/dist/pypy/translator/js/jsbuiltin.py Fri Sep 21 13:28:23 2007 @@ -29,6 +29,7 @@ 'll_time_time' : CallBuiltin('time'), 'll_time_clock' : CallBuiltin('clock'), 'll_os_write' : CallBuiltin('print'), + 'll_math.ll_math_pow' : CallBuiltin('Math.pow'), } self.builtin_obj_map = { ootype.String.__class__: { Modified: pypy/dist/pypy/translator/js/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_typed.py (original) +++ pypy/dist/pypy/translator/js/test/test_typed.py Fri Sep 21 13:28:23 2007 @@ -222,7 +222,7 @@ def func(x, y): z = x + y / 2.1 * x z = math.fmod(z, 60.0) - z = pow(z, 2) + z = math.pow(z, 2) z = -z return int(z) From arigo at codespeak.net Fri Sep 21 14:43:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 14:43:52 +0200 (CEST) Subject: [pypy-svn] r46790 - pypy/dist/pypy/rlib Message-ID: <20070921124352.52FC9817B@code0.codespeak.net> Author: arigo Date: Fri Sep 21 14:43:51 2007 New Revision: 46790 Modified: pypy/dist/pypy/rlib/rsocket_rffi.py Log: Translation fixes. Modified: pypy/dist/pypy/rlib/rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/rsocket_rffi.py Fri Sep 21 14:43:51 2007 @@ -37,10 +37,17 @@ pass -ntohs = _c.ntohs -ntohl = _c.ntohl -htons = _c.htons -htonl = _c.htonl +def ntohs(x): + return rffi.cast(lltype.Signed, _c.ntohs(x)) + +def ntohl(x): + return rffi.cast(lltype.Signed, _c.ntohl(x)) + +def htons(x): + return rffi.cast(lltype.Signed, _c.htons(x)) + +def htonl(x): + return rffi.cast(lltype.Signed, _c.htonl(x)) _FAMILIES = {} @@ -191,7 +198,7 @@ def __init__(self, host, port): makeipaddr(host, self) a = self.lock(_c.sockaddr_in) - a.c_sin_port = htons(port) + rffi.setintfield(a, 'c_sin_port', htons(port)) self.unlock() def __repr__(self): @@ -261,9 +268,9 @@ def __init__(self, host, port, flowinfo=0, scope_id=0): makeipaddr(host, self) a = self.lock(_c.sockaddr_in6) - a.c_sin6_port = htons(port) - a.c_sin6_flowinfo = flowinfo - a.c_sin6_scope_id = scope_id + rffi.setintfield(a, 'c_sin6_port', htons(port)) + rffi.setintfield(a, 'c_sin6_flowinfo', flowinfo) + rffi.setintfield(a, 'c_sin6_scope_id', scope_id) self.unlock() def __repr__(self): @@ -285,13 +292,13 @@ a = self.lock(_c.sockaddr_in6) flowinfo = a.c_sin6_flowinfo self.unlock() - return flowinfo + return rffi.cast(lltype.Unsigned, flowinfo) def get_scope_id(self): a = self.lock(_c.sockaddr_in6) scope_id = a.c_sin6_scope_id self.unlock() - return scope_id + return rffi.cast(lltype.Unsigned, scope_id) def eq(self, other): # __eq__() is not called by RPython :-/ return (isinstance(other, INET6Address) and @@ -428,13 +435,13 @@ a = self.lock(_c.sockaddr_nl) pid = a.c_nl_pid self.unlock() - return pid + return rffi.cast(lltype.Unsigned, pid) def get_groups(self): a = self.lock(_c.sockaddr_nl) groups = a.c_nl_groups self.unlock() - return groups + return rffi.cast(lltype.Unsigned, groups) def __repr__(self): return '' % (self.get_pid(), self.get_groups()) @@ -448,7 +455,7 @@ w_pid, w_groups = space.unpackiterable(w_address, 2) except ValueError: raise TypeError("AF_NETLINK address must be a tuple of length 2") - return NETLINKAddress(space.int_w(w_pid), space.int_w(w_groups)) + return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) # ____________________________________________________________ @@ -1113,7 +1120,7 @@ servent = _c.getservbyname(name, proto) if not servent: raise RSocketError("service/proto not found") - return _c.ntohs(servent.c_s_port) + return ntohs(servent.c_s_port) def getservbyport(port, proto=None): servent = _c.getservbyport(htons(port), proto) @@ -1125,7 +1132,8 @@ protoent = _c.getprotobyname(name) if not protoent: raise RSocketError("protocol not found") - return protoent.c_p_proto + proto = protoent.c_p_proto + return rffi.cast(lltype.Signed, proto) def getnameinfo(address, flags): host = lltype.malloc(rffi.CCHARP.TO, NI_MAXHOST, flavor='raw') From arigo at codespeak.net Fri Sep 21 14:54:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 14:54:47 +0200 (CEST) Subject: [pypy-svn] r46791 - in pypy/dist/pypy: module/_socket rlib rlib/test Message-ID: <20070921125447.6F01F817D@code0.codespeak.net> Author: arigo Date: Fri Sep 21 14:54:47 2007 New Revision: 46791 Added: pypy/dist/pypy/rlib/rsocket.py - copied unchanged from r46790, pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket.py - copied, changed from r46788, pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Removed: pypy/dist/pypy/rlib/_rsocket_ctypes.py pypy/dist/pypy/rlib/rsocket_rffi.py pypy/dist/pypy/rlib/test/test_rsocket_rffi.py Modified: pypy/dist/pypy/module/_socket/interp_func.py pypy/dist/pypy/module/_socket/interp_socket.py Log: Switch over to the rffi-based socket implementation. This breaks on Windows (and maybe on OS/X) but I plan to fix it soon again. When I have to use that Windows machine I really don't want all these easily-confused versions of the code around... Modified: pypy/dist/pypy/module/_socket/interp_func.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_func.py (original) +++ pypy/dist/pypy/module/_socket/interp_func.py Fri Sep 21 14:54:47 2007 @@ -10,12 +10,7 @@ Return the current host name. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - res = rsocket.gethostname() - finally: - if GIL is not None: GIL.acquire(True) + res = rsocket.gethostname() except SocketError, e: raise converted_error(space, e) return space.wrap(res) @@ -27,12 +22,7 @@ Return the IP address (a string of the form '255.255.255.255') for a host. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = rsocket.gethostbyname(hostname) - finally: - if GIL is not None: GIL.acquire(True) + addr = rsocket.gethostbyname(hostname) ip = addr.get_host() except SocketError, e: raise converted_error(space, e) @@ -53,12 +43,7 @@ for a host. The host argument is a string giving a host name or IP number. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - res = rsocket.gethostbyname_ex(host) - finally: - if GIL is not None: GIL.acquire(True) + res = rsocket.gethostbyname_ex(host) except SocketError, e: raise converted_error(space, e) return common_wrapgethost(space, res) @@ -71,12 +56,7 @@ for a host. The host argument is a string giving a host name or IP number. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - res = rsocket.gethostbyaddr(host) - finally: - if GIL is not None: GIL.acquire(True) + res = rsocket.gethostbyaddr(host) except SocketError, e: raise converted_error(space, e) return common_wrapgethost(space, res) @@ -94,12 +74,7 @@ else: proto = space.str_w(w_proto) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - port = rsocket.getservbyname(name, proto) - finally: - if GIL is not None: GIL.acquire(True) + port = rsocket.getservbyname(name, proto) except SocketError, e: raise converted_error(space, e) return space.wrap(port) @@ -117,12 +92,7 @@ else: proto = space.str_w(w_proto) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - service = rsocket.getservbyport(port, proto) - finally: - if GIL is not None: GIL.acquire(True) + service = rsocket.getservbyport(port, proto) except SocketError, e: raise converted_error(space, e) return space.wrap(service) @@ -134,12 +104,7 @@ Return the protocol number for the named protocol. (Rarely used.) """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - proto = rsocket.getprotobyname(name) - finally: - if GIL is not None: GIL.acquire(True) + proto = rsocket.getprotobyname(name) except SocketError, e: raise converted_error(space, e) return space.wrap(proto) @@ -150,14 +115,8 @@ Get host and port for a sockaddr.""" try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = rsocket.ipaddr_from_object(space, w_sockaddr) - host, servport = rsocket.getnameinfo(addr, flags) - finally: - if GIL is not None: GIL.acquire(True) - + addr = rsocket.ipaddr_from_object(space, w_sockaddr) + host, servport = rsocket.getnameinfo(addr, flags) except SocketError, e: raise converted_error(space, e) return space.newtuple([space.wrap(host), space.wrap(servport)]) @@ -327,14 +286,8 @@ raise OperationError(space.w_TypeError, space.wrap("Int or String expected")) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - lst = rsocket.getaddrinfo(host, port, family, socktype, - proto, flags) - finally: - if GIL is not None: GIL.acquire(True) - + lst = rsocket.getaddrinfo(host, port, family, socktype, + proto, flags) except SocketError, e: raise converted_error(space, e) lst1 = [space.newtuple([space.wrap(family), Modified: pypy/dist/pypy/module/_socket/interp_socket.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_socket.py (original) +++ pypy/dist/pypy/module/_socket/interp_socket.py Fri Sep 21 14:54:47 2007 @@ -30,12 +30,7 @@ info is a pair (hostaddr, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - sock, addr = self.accept(W_RSocket) - finally: - if GIL is not None: GIL.acquire(True) + sock, addr = self.accept(W_RSocket) return space.newtuple([space.wrap(sock), addr.as_object(space)]) except SocketError, e: @@ -50,12 +45,7 @@ sockets the address is a tuple (ifname, proto [,pkttype [,hatype]]) """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.bind(self.addr_from_object(space, w_addr)) - finally: - if GIL is not None: GIL.acquire(True) + self.bind(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) bind_w.unwrap_spec = ['self', ObjSpace, W_Root] @@ -66,12 +56,7 @@ Close the socket. It cannot be used after this call. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.close() - finally: - if GIL is not None: GIL.acquire(True) + self.close() except SocketError, e: raise converted_error(space, e) close_w.unwrap_spec = ['self', ObjSpace] @@ -83,12 +68,7 @@ is a pair (host, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.connect(self.addr_from_object(space, w_addr)) - finally: - if GIL is not None: GIL.acquire(True) + self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) except TypeError, e: @@ -102,12 +82,7 @@ This is like connect(address), but returns an error code (the errno value) instead of raising an exception when an error occurs. """ - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - error = self.connect_ex(self.addr_from_object(space, w_addr)) - finally: - if GIL is not None: GIL.acquire(True) + error = self.connect_ex(self.addr_from_object(space, w_addr)) return space.wrap(error) connect_ex_w.unwrap_spec = ['self', ObjSpace, W_Root] @@ -137,12 +112,7 @@ info is a pair (hostaddr, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = self.getpeername() - finally: - if GIL is not None: GIL.acquire(True) + addr = self.getpeername() return addr.as_object(space) except SocketError, e: raise converted_error(space, e) @@ -155,12 +125,7 @@ info is a pair (hostaddr, port). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = self.getsockname() - finally: - if GIL is not None: GIL.acquire(True) + addr = self.getsockname() return addr.as_object(space) except SocketError, e: raise converted_error(space, e) @@ -202,12 +167,7 @@ will allow before refusing new connections. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.listen(backlog) - finally: - if GIL is not None: GIL.acquire(True) + self.listen(backlog) except SocketError, e: raise converted_error(space, e) listen_w.unwrap_spec = ['self', ObjSpace, int] @@ -232,13 +192,7 @@ the remote end is closed and all data is read, return the empty string. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - data = self.recv(buffersize, flags) - finally: - if GIL is not None: GIL.acquire(True) - + data = self.recv(buffersize, flags) except SocketError, e: raise converted_error(space, e) return space.wrap(data) @@ -250,12 +204,7 @@ Like recv(buffersize, flags) but also return the sender's address info. """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - data, addr = self.recvfrom(buffersize, flags) - finally: - if GIL is not None: GIL.acquire(True) + data, addr = self.recvfrom(buffersize, flags) if addr: w_addr = addr.as_object(space) else: @@ -274,12 +223,7 @@ """ data = coerce_to_str_w(space, w_data) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - count = self.send(data, flags) - finally: - if GIL is not None: GIL.acquire(True) + count = self.send(data, flags) except SocketError, e: raise converted_error(space, e) return space.wrap(count) @@ -295,12 +239,7 @@ """ data = coerce_to_str_w(space, w_data) try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - count = self.sendall(data, flags) - finally: - if GIL is not None: GIL.acquire(True) + count = self.sendall(data, flags) except SocketError, e: raise converted_error(space, e) sendall_w.unwrap_spec = ['self', ObjSpace, W_Root, int] @@ -321,13 +260,8 @@ flags = space.int_w(w_param2) w_addr = w_param3 try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - addr = self.addr_from_object(space, w_addr) - count = self.sendto(data, flags, addr) - finally: - if GIL is not None: GIL.acquire(True) + addr = self.addr_from_object(space, w_addr) + count = self.sendto(data, flags, addr) except SocketError, e: raise converted_error(space, e) return space.wrap(count) @@ -391,12 +325,7 @@ (flag == SHUT_RDWR). """ try: - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - try: - self.shutdown(how) - finally: - if GIL is not None: GIL.acquire(True) + self.shutdown(how) except SocketError, e: raise converted_error(space, e) shutdown_w.unwrap_spec = ['self', ObjSpace, int] From arigo at codespeak.net Fri Sep 21 15:09:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 15:09:02 +0200 (CEST) Subject: [pypy-svn] r46792 - pypy/dist/pypy/module/select Message-ID: <20070921130902.58F34818B@code0.codespeak.net> Author: arigo Date: Fri Sep 21 15:08:59 2007 New Revision: 46792 Added: pypy/dist/pypy/module/select/_rsocket_ctypes.py - copied unchanged from r46790, pypy/dist/pypy/rlib/_rsocket_ctypes.py Modified: pypy/dist/pypy/module/select/__init__.py pypy/dist/pypy/module/select/interp_select.py Log: Temporarily resurrect _rsocket_ctypes into module/select/ which needs it. Modified: pypy/dist/pypy/module/select/__init__.py ============================================================================== --- pypy/dist/pypy/module/select/__init__.py (original) +++ pypy/dist/pypy/module/select/__init__.py Fri Sep 21 15:08:59 2007 @@ -17,7 +17,7 @@ POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG'''.split() - from pypy.rlib._rsocket_ctypes import constants + from _rsocket_ctypes import constants for name in constantnames: if name in constants: value = constants[name] Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Fri Sep 21 15:08:59 2007 @@ -1,7 +1,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import W_Root, ObjSpace, interp2app -from pypy.rlib import _rsocket_ctypes as _c +import _rsocket_ctypes as _c from ctypes import POINTER, byref from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError From cfbolz at codespeak.net Fri Sep 21 15:27:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 21 Sep 2007 15:27:36 +0200 (CEST) Subject: [pypy-svn] r46793 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20070921132736.AFAE68188@code0.codespeak.net> Author: cfbolz Date: Fri Sep 21 15:27:34 2007 New Revision: 46793 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Log: make test_ll2ctypes.py pass on OS X Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Fri Sep 21 15:27:34 2007 @@ -624,6 +624,6 @@ return standard_c_lib.__errno_location() elif sys.platform == 'darwin': - standard_c_lib.__errno.restype = ctypes.POINTER(ctypes.c_int) + standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): - return standard_c_lib.__errno() + return standard_c_lib.__error() From cfbolz at codespeak.net Fri Sep 21 15:40:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 21 Sep 2007 15:40:05 +0200 (CEST) Subject: [pypy-svn] r46795 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20070921134005.75C0E818D@code0.codespeak.net> Author: cfbolz Date: Fri Sep 21 15:40:04 2007 New Revision: 46795 Added: pypy/extradoc/sprintinfo/bern2007/ pypy/extradoc/sprintinfo/bern2007/announce.txt pypy/extradoc/sprintinfo/bern2007/people.txt Log: (Adrian Kuhn, cfbolz): first draft of Bern Squeak/PyPy sprint announcement Added: pypy/extradoc/sprintinfo/bern2007/announce.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/bern2007/announce.txt Fri Sep 21 15:40:04 2007 @@ -0,0 +1,63 @@ +Bern Squeak/PyPy sprint October 22 - October 26 2007 +===================================================== + +We are proud to announce the first Squeak-PyPy-collaboration-sprint. +The plan is to bring people from the Squeak/Smalltalk and the +PyPy/Python communities together to learn about each others projects and +to think about collaboration possibilies. The sprint will be hosted by +the Software Composition Group of the University of Bern from the 22nd +to the 26th of October 2007. + + +Topics and goals +---------------- + +The general technical goal of the sprint is to explore implementing the +bytecode interpreter as well as the primitives of Squeak in RPython. +This should make it eventually possible to auto-generate a JIT for +Squeak with the help of PyPy technologies. + +The other important goal of the sprint is to teach the +first-time-sprinters the sprint-driven-development methodology. + +The first day of the sprint will be a tutorial day, were talks about +PyPy, Squeak and sprint-driven-development will be given. The further +days will be sprinting days. + +Of course the most important goal of the sprint is to get to know each +other and have some fun :-). + +Topics of the sprint: + +- learning about: + + - PyPy + - the internals of Squeaks VM + - sprint-driven-development + +- exploring ways to implement Squeaks bytecode-interpreter and the + primitives in Python + +- think about how to load and store of images in PyPy + + +Location +-------- + +The sprint will take place in a seminar room at the University of Bern. +Further details of how to get there will be announced before the sprint. + + +Registration +------------ + +If you'd like to come, please subscribe to the `pypy-sprint mailing list`_ +and drop a note about your interests and post any questions. More +organisational information will be send to that list. We'll keep a list +of `people`_ which we'll update (which you can do so yourself if you +have codespeak commit rights). If you have no clue where to stay in +Bern ask around on the list, maybe we can help you find something or +find some other solution. + +.. _`pypy-sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint +.. _`people`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/people.html Added: pypy/extradoc/sprintinfo/bern2007/people.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/bern2007/people.txt Fri Sep 21 15:40:04 2007 @@ -0,0 +1,51 @@ +People coming to the Bern sprint October 2007 +==================================================== + +People who have a ``?`` in their arrive/depart or accomodation +column are known to be coming but there are no details +available yet from them. + +==================== ============== ===================== + Name Arrive/Depart Accomodation +==================== ============== ===================== +Carl Friedrich Bolz ? ? +Adrian Kuhn - private +==================== ============== ===================== + + +People on the following list were present at previous sprints: + +==================== ============== ===================== + Name Arrive/Depart Accomodation +==================== ============== ===================== +Armin Rigo ? ? +Michael Hudson ? ? +Antonio Cuni ? ? +Samuele Pedroni ? ? +Anders Chrigstroem ? ? +Maciej Fijalkowski ? ? +Andrew Thompson ? ? +Stephan Diehl ? ? +Eric van Riet paap ? ? +Niko Matsakis ? ? +Leonardo Santagada ? ? +Richard Emslie ? ? +Christian Tismer ? ? +Alexander Schremmer ? ? +Jacob Hallen ? ? +Aurelien Campeas ? ? +Alexandre Fayolle ? ? +Lene Wagner ? ? +Amaury Forgeot d'Arc ? ? +Valentino Volonghi ? ? +Boris Feigin ? ? +Bert Freudenberg ? ? +Laura Creighton ? ? +Beatrice Duering ? ? +Johan Hahn ? ? +Holger Krekel ? ? +Anders Chrigstroem ? ? +Samuele Pedroni ? ? +Anders Lehmann ? ? +Niklaus Haldimann ? ? +==================== ============== ===================== From antocuni at codespeak.net Fri Sep 21 15:56:53 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 21 Sep 2007 15:56:53 +0200 (CEST) Subject: [pypy-svn] r46796 - in pypy/dist/pypy: rpython/ootypesystem rpython/ootypesystem/test translator/backendopt translator/backendopt/test Message-ID: <20070921135653.BCEFB8193@code0.codespeak.net> Author: antocuni Date: Fri Sep 21 15:56:51 2007 New Revision: 46796 Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py pypy/dist/pypy/translator/backendopt/canraise.py pypy/dist/pypy/translator/backendopt/graphanalyze.py pypy/dist/pypy/translator/backendopt/test/test_canraise.py pypy/dist/pypy/translator/backendopt/test/test_inline.py Log: properly analyze oosends. This makes test_canraise.test_method and test_inline.test_list_iteration passing again. Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Fri Sep 21 15:56:51 2007 @@ -1,4 +1,5 @@ import py +from py.builtin import set from pypy.rpython.lltypesystem.lltype import LowLevelType, Signed, Unsigned, Float, Char from pypy.rpython.lltypesystem.lltype import Bool, Void, UniChar, typeOf, \ Primitive, isCompatibleType, enforce, saferecursive, SignedLongLong, UnsignedLongLong @@ -194,6 +195,15 @@ all.update(self._fields) return all + def _lookup_graphs(self, meth_name): + _, meth = self._lookup(meth_name) + graphs = set() + graphs.add(meth.graph) # we assume there is always a graph + for SUBTYPE in self._subclasses: + graphs.update(SUBTYPE._lookup_graphs(meth_name)) + return graphs + + class SpecializableType(OOType): def _specialize_type(self, TYPE, generic_types): if isinstance(TYPE, SpecializableType): @@ -308,6 +318,9 @@ meth._virtual = False return self, meth + def _lookup_graphs(self, meth_name): + return set() + # WARNING: the name 'String' is rebound at the end of file class String(BuiltinADTType): Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py Fri Sep 21 15:56:51 2007 @@ -475,3 +475,37 @@ def test_dead_wref(): ref = new(WeakReference) assert ref.ll_deref() is null(ROOT) + + +# we use the translator for this test because it's much easier to get +# a class hierarchy with methods and graphs by using it than +# constructing it by hand +def test_lookup_graphs(): + from pypy.translator.translator import TranslationContext, graphof + class A: + def foo(self): + pass + def bar(self): + pass + + class B(A): + def foo(self): + pass + + def fn(flag): + obj = flag and A() or B() + obj.foo() + obj.bar() + return obj + + t = TranslationContext() + t.buildannotator().build_types(fn, [int]) + t.buildrtyper(type_system='ootype').specialize() + graph = graphof(t, fn) + TYPE_A = graph.getreturnvar().concretetype + TYPE_B = TYPE_A._subclasses[0] + assert len(TYPE_A._lookup_graphs('ofoo')) == 2 + assert len(TYPE_B._lookup_graphs('ofoo')) == 1 + assert len(TYPE_A._lookup_graphs('obar')) == 1 + assert len(TYPE_B._lookup_graphs('obar')) == 1 + Modified: pypy/dist/pypy/translator/backendopt/canraise.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/canraise.py (original) +++ pypy/dist/pypy/translator/backendopt/canraise.py Fri Sep 21 15:56:51 2007 @@ -16,12 +16,15 @@ log.WARNING("Unknown operation: %s" % op.opname) return True - def analyze_external_call(self, op): deref = self.translator.rtyper.type_system_deref fnobj = deref(op.args[0].value) return getattr(fnobj, 'canraise', True) + def analyze_external_method(self, op, TYPE, meth): + assert op.opname == 'oosend' + return getattr(meth, '_can_raise', True) + def analyze_exceptblock(self, block, seen=None): return True Modified: pypy/dist/pypy/translator/backendopt/graphanalyze.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/graphanalyze.py (original) +++ pypy/dist/pypy/translator/backendopt/graphanalyze.py Fri Sep 21 15:56:51 2007 @@ -23,6 +23,9 @@ def analyze_external_call(self, op): return True + def analyze_external_method(self, op, TYPE, meth): + return True + def analyze_link(self, graph, link): return False @@ -38,6 +41,14 @@ if op.args[-1].value is None: return True return self.analyze_indirect_call(op.args[-1].value, seen) + elif op.opname == "oosend": + name = op.args[0].value + TYPE = op.args[1].concretetype + _, meth = TYPE._lookup(name) + graph = getattr(meth, 'graph', None) + if graph is None: + return self.analyze_external_method(op, TYPE, meth) + return self.analyze_oosend(TYPE, name, seen=None) if self.operation_is_true(op): return True @@ -77,6 +88,10 @@ return True return False + def analyze_oosend(self, TYPE, name, seen=None): + graphs = TYPE._lookup_graphs(name) + return self.analyze_indirect_call(graphs, seen) + def analyze_all(self, graphs=None): if graphs is None: graphs = self.translator.graphs Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_canraise.py Fri Sep 21 15:56:51 2007 @@ -1,4 +1,5 @@ from pypy.translator.translator import TranslationContext, graphof +from pypy.translator.simplify import get_funcobj from pypy.translator.backendopt.canraise import RaiseAnalyzer from pypy.translator.backendopt.all import backend_optimizations from pypy.rpython.test.tool import LLRtypeMixin, OORtypeMixin @@ -6,9 +7,6 @@ class BaseTestCanRaise(object): type_system = None - def _skip_oo(self, reason): - if self.type_system == 'ootype': - py.test.skip("ootypesystem doesn't support %s, yet" % reason) def translate(self, func, sig): t = TranslationContext() @@ -74,26 +72,44 @@ assert result def test_method(self): - self._skip_oo("oosend analysis") class A(object): - def f(x): + def f(self): return 1 + def m(self): + raise ValueError class B(A): - def f(x): + def f(self): return 2 + def m(self): + return 3 def f(a): return a.f() - def h(x): - if x: - a = A() + def m(a): + return a.m() + def h(flag): + if flag: + obj = A() else: - a = B() - return f(a) + obj = B() + f(obj) + m(obj) + t, ra = self.translate(h, [int]) hgraph = graphof(t, h) # fiiiish :-( - result = ra.can_raise(hgraph.startblock.exits[0].target.exits[0].target.operations[0]) - assert not result + block = hgraph.startblock.exits[0].target.exits[0].target + op_call_f = block.operations[0] + op_call_m = block.operations[1] + + # check that we fished the expected ops + def check_call(op, fname): + assert op.opname == "direct_call" + assert get_funcobj(op.args[0].value)._name == fname + check_call(op_call_f, "f") + check_call(op_call_m, "m") + + assert not ra.can_raise(op_call_f) + assert ra.can_raise(op_call_m) def test_instantiate(self): # instantiate is interesting, because it leads to one of the few cases of Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_inline.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_inline.py Fri Sep 21 15:56:51 2007 @@ -544,7 +544,6 @@ py.test.raises(CannotInline, self.check_inline, x3, x4, []) def test_list_iteration(self): - self._skip_oo("graphanalyze properly!") def f(): tot = 0 for item in [1,2,3]: From arigo at codespeak.net Fri Sep 21 16:05:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 16:05:54 +0200 (CEST) Subject: [pypy-svn] r46797 - pypy/dist/pypy/rlib Message-ID: <20070921140554.8AC8D8195@code0.codespeak.net> Author: arigo Date: Fri Sep 21 16:05:53 2007 New Revision: 46797 Modified: pypy/dist/pypy/rlib/rsocket.py Log: Some more OS/X compatibility. Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Fri Sep 21 16:05:53 2007 @@ -1083,10 +1083,11 @@ address_to_fill=None): # port_or_service is a string, not an int (but try str(port_number)). assert port_or_service is None or isinstance(port_or_service, str) - hints = rffi.make(_c.addrinfo, c_ai_family = family, - c_ai_socktype = socktype, - c_ai_protocol = proto, - c_ai_flags = flags) + hints = lltype.malloc(_c.addrinfo, flavor='raw', zero=True) + rffi.setintfield(hints, 'c_ai_family', family) + rffi.setintfield(hints, 'c_ai_socktype', socktype) + rffi.setintfield(hints, 'c_ai_protocol', proto) + rffi.setintfield(hints, 'c_ai_flags' , flags) # XXX need to lock around getaddrinfo() calls? p_res = lltype.malloc(rffi.CArray(_c.addrinfo_ptr), 1, flavor='raw') error = _c.getaddrinfo(host, port_or_service, hints, p_res) From arigo at codespeak.net Fri Sep 21 16:43:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 16:43:39 +0200 (CEST) Subject: [pypy-svn] r46798 - pypy/dist/pypy/rlib Message-ID: <20070921144339.516138131@code0.codespeak.net> Author: arigo Date: Fri Sep 21 16:43:38 2007 New Revision: 46798 Modified: pypy/dist/pypy/rlib/rsocket.py Log: Zero-initialize all sockaddr structures. Helps at least on OS/X. Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Fri Sep 21 16:43:38 2007 @@ -245,7 +245,7 @@ result = instantiate(INETAddress) # store the malloc'ed data into 'result' as soon as possible # to avoid leaks if an exception occurs inbetween - sin = rffi.make(_c.sockaddr_in) + sin = lltype.malloc(_c.sockaddr_in, flavor='raw', zero=True) result.setdata(sin, sizeof(_c.sockaddr_in)) # PLAT sin_len rffi.setintfield(sin, 'c_sin_family', AF_INET) @@ -348,7 +348,7 @@ result = instantiate(INET6Address) # store the malloc'ed data into 'result' as soon as possible # to avoid leaks if an exception occurs inbetween - sin = rffi.make(_c.sockaddr_in6) + sin = lltype.malloc(_c.sockaddr_in6, flavor='raw', zero=True) result.setdata(sin, sizeof(_c.sockaddr_in6)) rffi.setintfield(sin, 'c_sin6_family', AF_INET) rffi.structcopy(sin.c_sin6_addr, in6_addr) @@ -370,7 +370,7 @@ maxlen = sizeof(struct) def __init__(self, path): - sun = rffi.make(_c.sockaddr_un) + sun = lltype.malloc(_c.sockaddr_un, flavor='raw', zero=True) baseofs = offsetof(_c.sockaddr_un, 'c_sun_path') self.setdata(sun, baseofs + len(path)) rffi.setintfield(sun, 'c_sun_family', AF_UNIX) @@ -425,7 +425,7 @@ maxlen = minlen = sizeof(struct) def __init__(self, pid, groups): - addr = rffi.make(_c.sockaddr_nl) + addr = lltype.malloc(_c.sockaddr_nl, flavor='raw', zero=True) self.setdata(addr, NETLINKAddress.maxlen) rffi.setintfield(addr, 'c_nl_family', AF_NETLINK) rffi.setintfield(addr, 'c_nl_pid', pid) @@ -484,7 +484,7 @@ result = instantiate(INETAddress) elif result.family != AF_INET: raise RSocketError("address family mismatched") - sin = rffi.make(_c.sockaddr_in) + sin = lltype.malloc(_c.sockaddr_in, flavor='raw', zero=True) result.setdata(sin, sizeof(_c.sockaddr_in)) rffi.setintfield(sin, 'c_sin_family', AF_INET) # PLAT sin_len rffi.setintfield(sin.c_sin_addr, 'c_s_addr', s_addr) @@ -493,7 +493,7 @@ def make_null_address(family): klass = familyclass(family) result = instantiate(klass) - buf = mallocbuf(klass.maxlen) + buf = lltype.malloc(rffi.CCHARP.TO, klass.maxlen, flavor='raw', zero=True) result.setdata(buf, 0) return result, klass.maxlen From arigo at codespeak.net Fri Sep 21 16:49:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Sep 2007 16:49:46 +0200 (CEST) Subject: [pypy-svn] r46799 - pypy/dist/pypy/rlib/test Message-ID: <20070921144946.C370E8131@code0.codespeak.net> Author: arigo Date: Fri Sep 21 16:49:46 2007 New Revision: 46799 Modified: pypy/dist/pypy/rlib/test/test_rsocket.py Log: Try to never perform the lookup, with this flag. Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Fri Sep 21 16:49:46 2007 @@ -250,7 +250,7 @@ def test_getaddrinfo_no_reverse_lookup(): # It seems that getaddrinfo never runs a reverse lookup on Linux. # Python2.3 on Windows returns the hostname. - lst = getaddrinfo('134.99.112.214', None, flags=AI_CANONNAME) + lst = getaddrinfo('134.99.112.214', None, flags=AI_NUMERICHOST) assert isinstance(lst, list) found = False for family, socktype, protocol, canonname, addr in lst: From cfbolz at codespeak.net Sat Sep 22 09:17:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 22 Sep 2007 09:17:46 +0200 (CEST) Subject: [pypy-svn] r46809 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20070922071746.2FEF2810C@code0.codespeak.net> Author: cfbolz Date: Sat Sep 22 09:17:44 2007 New Revision: 46809 Modified: pypy/extradoc/sprintinfo/bern2007/announce.txt Log: add the Squeak backend as a task too Modified: pypy/extradoc/sprintinfo/bern2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/announce.txt Sat Sep 22 09:17:44 2007 @@ -38,6 +38,8 @@ - exploring ways to implement Squeaks bytecode-interpreter and the primitives in Python +- or (maybe alternatively) implement a Squeak-backend for PyPy + - think about how to load and store of images in PyPy From cfbolz at codespeak.net Sat Sep 22 09:32:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 22 Sep 2007 09:32:11 +0200 (CEST) Subject: [pypy-svn] r46810 - pypy/dist/pypy/doc Message-ID: <20070922073211.38985810D@code0.codespeak.net> Author: cfbolz Date: Sat Sep 22 09:32:10 2007 New Revision: 46810 Modified: pypy/dist/pypy/doc/news.txt Log: add news item about the squeak sprint Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Sat Sep 22 09:32:10 2007 @@ -14,13 +14,30 @@ .. _`iCalendar format`: webcal://pypycal.sabi.net///calendars/PyPy.ics .. _eventhistory: eventhistory.html -PyPy Sprint at EuroPython, Vilnius: 12-14th July 2007 + +PyPy/Squeak Sprint in Bern: 22nd-26th October 2007 +================================================================== + +We are proud to announce the first Squeak-PyPy-collaboration-sprint. The plan +is to bring people from the Squeak/Smalltalk and the PyPy/Python communities +together to learn about each others projects and to think about collaboration +possibilies. The sprint will be hosted by the Software Composition Group of +the University of Bern from the 22nd to the 26th of October 2007. For more +information see the `Bern sprint announcement`_ or a list of the `people that +are known to come`_. + +.. _`Bern sprint announcement`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/announce.html +.. _`people that are known to come`: http://codespeak.net/pypy/extradoc/sprintinfo/bern2007/people.html + + + +PyPy Sprint at EuroPython, Vilnius is finished ================================================================== -The next EuroPython_ conference will be held in Vilnius from the 9th to -the 11th of July, 2007. It now became a tradition to have several PyPy -talks during the conference, followed by a sprint. For more information -on the latter, see the `Vilnius sprint announcement`_. +The sprint at the last EuroPython_ conference in Vilnius from the 9th to +the 11th of July, 2007 is finished. For more information +see the `Vilnius sprint announcement`_. + .. _EuroPython: http://europython.org .. _`Vilnius sprint announcement`: http://codespeak.net/pypy/extradoc/sprintinfo/post-ep2007/announcement.html @@ -76,7 +93,7 @@ -PyPy Trillke Sprints (25-28th Feb and 1-5th March 2007) finished, +PyPy Trillke Sprints (25-28th Feb and 1-5th March 2007) finished ================================================================== Both of the sprints that mark the end of the EU period are over. There were very From arigo at codespeak.net Sat Sep 22 11:03:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 11:03:01 +0200 (CEST) Subject: [pypy-svn] r46811 - in pypy/dist/pypy: annotation rlib rlib/test rpython Message-ID: <20070922090301.64E908103@code0.codespeak.net> Author: arigo Date: Sat Sep 22 11:02:59 2007 New Revision: 46811 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/rlib/rbigint.py pypy/dist/pypy/rlib/test/test_rbigint.py pypy/dist/pypy/rpython/rint.py Log: Remove this "very temporary hack". This creates a path in rbigint that the annotator doesn't know is unreachable, but on the other hand the hack in the annotator was really getting in the way (see test in the next checkin). Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Sat Sep 22 11:02:59 2007 @@ -299,12 +299,6 @@ r = SomeBool() if int1.is_immutable_constant() and int2.is_immutable_constant(): r.const = operation(int1.const, int2.const) - else: - # XXX VERY temporary hack - if (opname == 'ge' and int2.is_immutable_constant() and - int2.const == 0 and - not rarithmetic.signedtype(int1.knowntype)): - r.const = True knowntypedata = {} # XXX HACK HACK HACK # propagate nonneg information between the two arguments Modified: pypy/dist/pypy/rlib/rbigint.py ============================================================================== --- pypy/dist/pypy/rlib/rbigint.py (original) +++ pypy/dist/pypy/rlib/rbigint.py Sat Sep 22 11:02:59 2007 @@ -610,11 +610,10 @@ digits_for_most_neg_long._annspecialcase_ = "specialize:argtype(0)" def args_from_rarith_int(x): - if x >= 0: - if x == 0: - return [0], 0 - else: - return digits_from_nonneg_long(x), 1 + if x > 0: + return digits_from_nonneg_long(x), 1 + elif x == 0: + return [0], 0 else: try: y = ovfcheck(-x) Modified: pypy/dist/pypy/rlib/test/test_rbigint.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rbigint.py (original) +++ pypy/dist/pypy/rlib/test/test_rbigint.py Sat Sep 22 11:02:59 2007 @@ -424,11 +424,15 @@ def test_args_from_rarith_int(self): from pypy.rpython.test.test_llinterp import interpret - def fn(): - return (rbigint.fromrarith_int(0), - rbigint.fromrarith_int(17), - rbigint.fromrarith_int(-17), - rbigint.fromrarith_int(r_uint(0)), - rbigint.fromrarith_int(r_uint(17))) - interpret(fn, []) - + def fn(x): + n1 = rbigint.fromrarith_int(x) + n2 = rbigint.fromrarith_int(r_uint(x)) + return '%s %s' % (n1.str(), n2.str()) + res = interpret(fn, [0]) + assert ''.join(res.chars) == '0 0' + res = interpret(fn, [sys.maxint]) + assert ''.join(res.chars) == '%d %d' % (sys.maxint, sys.maxint) + res = interpret(fn, [-sys.maxint-1]) + assert ''.join(res.chars) == '%d %d' % (-sys.maxint-1, sys.maxint+1) + res = interpret(fn, [-17]) + assert ''.join(res.chars) == '%d %d' % (-17, 2*sys.maxint+2-17) Modified: pypy/dist/pypy/rpython/rint.py ============================================================================== --- pypy/dist/pypy/rpython/rint.py (original) +++ pypy/dist/pypy/rpython/rint.py Sat Sep 22 11:02:59 2007 @@ -346,14 +346,17 @@ if hop.s_result.unsigned: zero = self.lowleveltype._defl() vlist.insert(0, hop.inputconst(self.lowleveltype, zero)) - return hop.genop(self.opprefix + 'sub', vlist, resulttype=self) + return hop.genop(self.opprefix + 'neg', vlist, resulttype=self) else: return hop.genop(self.opprefix + 'neg', vlist, resulttype=self) def rtype_neg_ovf(self, hop): self = self.as_int if hop.s_result.unsigned: - raise TyperError("forbidden uint_neg_ovf") + # this is supported (and turns into just uint_neg) for + # rbigint.py + hop.exception_cannot_occur() + return self.rtype_neg(hop) else: vlist = hop.inputargs(self) hop.has_implicit_exception(OverflowError) # record we know about it From arigo at codespeak.net Sat Sep 22 11:12:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 11:12:22 +0200 (CEST) Subject: [pypy-svn] r46812 - in pypy/dist/pypy/annotation: . test Message-ID: <20070922091222.4A0B7810D@code0.codespeak.net> Author: arigo Date: Sat Sep 22 11:12:20 2007 New Revision: 46812 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: Test and fix for a translation issue that shows up in module/_socket. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Sat Sep 22 11:12:20 2007 @@ -299,9 +299,22 @@ r = SomeBool() if int1.is_immutable_constant() and int2.is_immutable_constant(): r.const = operation(int1.const, int2.const) + # + # The rest of the code propagates nonneg information between + # the two arguments. + # + # Doing the right thing when int1 or int2 change from signed + # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. + # Instead, we only deduce constrains on the operands in the + # case where they are both signed. In other words, if y is + # nonneg then "assert x>=y" will let the annotator know that + # x is nonneg too, but it will not work if y is unsigned. + # + if not (rarithmetic.signedtype(int1.knowntype) and + rarithmetic.signedtype(int2.knowntype)): + return r knowntypedata = {} # XXX HACK HACK HACK - # propagate nonneg information between the two arguments fn, block, i = getbookkeeper().position_key op = block.operations[i] assert op.opname == opname Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Sat Sep 22 11:12:20 2007 @@ -2839,7 +2839,14 @@ a = self.RPythonAnnotator() py.test.raises(NotImplementedError, a.build_types, f, [float, float]) - + def test_intcmp_bug(self): + def g(x, y): + return x <= y + def f(x, y): + if g(x, y): + g(x, r_uint(y)) + a = self.RPythonAnnotator() + a.build_types(f, [int, int]) def g(n): From arigo at codespeak.net Sat Sep 22 12:07:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 12:07:58 +0200 (CEST) Subject: [pypy-svn] r46813 - in pypy/dist/pypy: rpython rpython/memory/gctransform translator/c/test Message-ID: <20070922100758.43B0C80F7@code0.codespeak.net> Author: arigo Date: Sat Sep 22 12:07:55 2007 New Revision: 46813 Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/translator/c/test/test_lltyped.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: Support for malloc(flavor='raw', zero=True). The gctransformer inserts a 'raw_memclear' operation in the graph to handle this case. Not implemented: varsized raw zero mallocs. Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Sat Sep 22 12:07:55 2007 @@ -343,12 +343,14 @@ flags = hop.spaceop.args[1].value flavor = flags['flavor'] assert flavor == 'raw' + assert not flags.get('zero') return self.parenttransformer.gct_malloc(hop) def gct_malloc_varsize(self, hop): flags = hop.spaceop.args[1].value flavor = flags['flavor'] assert flavor == 'raw' + assert not flags.get('zero') return self.parenttransformer.gct_malloc_varsize(hop) def gct_free(self, hop): @@ -435,17 +437,22 @@ def gct_fv_raw_malloc(self, hop, flags, TYPE, c_size): v_raw = hop.genop("direct_call", [self.raw_malloc_fixedsize_ptr, c_size], resulttype=llmemory.Address) + if flags.get('zero'): + hop.genop("raw_memclear", [v_raw, c_size]) return v_raw def gct_fv_stack_malloc(self, hop, flags, TYPE, c_size): v_raw = hop.genop("direct_call", [self.stack_malloc_fixedsize_ptr, c_size], resulttype=llmemory.Address) + if flags.get('zero'): + hop.genop("raw_memclear", [v_raw, c_size]) return v_raw def gct_fv_cpy_malloc(self, hop, flags, TYPE, c_size): # xxx op = hop.spaceop args = op.args[:] del args[1] + assert not flags.get('zero') return hop.genop('cpy_malloc', args, resulttype=op.result.concretetype) def gct_malloc_varsize(self, hop): @@ -497,6 +504,8 @@ [self.raw_malloc_varsize_ptr, v_length, c_const_size, c_item_size, c_offset_to_length], resulttype=llmemory.Address) + if flags.get('zero'): + raise NotImplementedError("raw zero varsize malloc") return v_raw def gct_free(self, hop): Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Sat Sep 22 12:07:55 2007 @@ -335,7 +335,7 @@ if v_flavor is not None: flags['flavor'] = v_flavor.value if i_zero is not None: - assert i_extra_args is i_flavor is None + assert i_extra_args is None flags['zero'] = v_zero.value vlist.append(hop.inputconst(lltype.Void, flags)) Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Sat Sep 22 12:07:55 2007 @@ -416,6 +416,22 @@ res = fn(100) assert res == 3050 + def test_zero_raw_malloc(self): + S = Struct('S', ('x', Signed), ('y', Signed)) + def f(n): + for i in range(n): + p = malloc(S, flavor='raw', zero=True) + if p.x != 0 or p.y != 0: + return -1 + p.x = i + p.y = i + free(p, flavor='raw') + return 42 + + fn = self.getcompiled(f, [int]) + res = fn(100) + assert res == 42 + def test_arithmetic_cornercases(self): import operator, sys from pypy.rlib.unroll import unrolling_iterable Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Sat Sep 22 12:07:55 2007 @@ -758,6 +758,22 @@ # the point is just not to segfault res = fn() + def test_zero_raw_malloc(self): + S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + def f(): + for i in range(100): + p = lltype.malloc(S, flavor='raw', zero=True) + if p.x != 0 or p.y != 0: + return -1 + p.x = i + p.y = i + lltype.free(p, flavor='raw') + return 42 + + fn = self.getcompiled(f) + res = fn() + assert res == 42 + class TestUsingStacklessFramework(TestUsingFramework): gcpolicy = "stacklessgc" From arigo at codespeak.net Sat Sep 22 12:36:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 12:36:05 +0200 (CEST) Subject: [pypy-svn] r46814 - in pypy/dist/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20070922103605.5DDBA8110@code0.codespeak.net> Author: arigo Date: Sat Sep 22 12:36:04 2007 New Revision: 46814 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/getaddrinfo.py pypy/dist/pypy/rlib/getnameinfo.py pypy/dist/pypy/rlib/rsocket.py pypy/dist/pypy/rlib/test/test_rsocket.py pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Progress towards Windows support in rsocket. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Sat Sep 22 12:36:04 2007 @@ -29,11 +29,15 @@ 'errno.h', ) cond_includes = [('AF_NETLINK', 'linux/netlink.h')] + libraries = () + calling_conv = 'c' HEADER = ''.join(['#include <%s>\n' % filename for filename in includes]) COND_HEADER = ''.join(['#ifdef %s\n#include <%s>\n#endif\n' % cond_include for cond_include in cond_includes]) if _MS_WINDOWS: includes = ('WinSock2.h', 'WS2tcpip.h') + libraries = ('ws2_32',) + calling_conv = 'win' HEADER = '\n'.join([ '#include ', '#include ', @@ -295,14 +299,14 @@ # HACK HACK HACK if _MS_WINDOWS: - XXX - from ctypes import Structure - for struct in cConfig.__dict__.values(): - if isinstance(struct, type) and issubclass(struct, Structure): - if struct.__name__ == 'in6_addr': - struct.__name__ = '_in6_addr' - else: - struct._external_ = True # hack to avoid redeclaration of the struct in C + pass #XXX + #from ctypes import Structure + #for struct in cConfig.__dict__.values(): + # if isinstance(struct, type) and issubclass(struct, Structure): + # if struct.__name__ == 'in6_addr': + # struct.__name__ = '_in6_addr' + # else: + # struct._external_ = True # hack to avoid redeclaration of the struct in C # fill in missing constants with reasonable defaults cConfig.NI_MAXHOST = cConfig.NI_MAXHOST or 1025 @@ -385,7 +389,9 @@ includes.append(_header) def external(name, args, result): - return rffi.llexternal(name, args, result, includes=includes) + return rffi.llexternal(name, args, result, + includes=includes, libraries=libraries, + calling_conv=calling_conv) if _POSIX: dup = external('dup', [socketfd_type], socketfd_type) @@ -399,7 +405,10 @@ socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type) -socketclose = external('close', [socketfd_type], rffi.INT) +if MS_WINDOWS: + socketclose = external('closesocket', [socketfd_type], rffi.INT) +else: + socketclose = external('close', [socketfd_type], rffi.INT) socketconnect = external('connect', [socketfd_type, sockaddr_ptr, socklen_t], rffi.INT) @@ -468,53 +477,46 @@ lltype.Ptr(socketpair_t)], rffi.INT) if _MS_WINDOWS: - XXX - ioctlsocket = socketdll.ioctlsocket - ioctlsocket.argtypes = [socketfd_type, rffi.LONG, POINTER(c_ulong)] - ioctlsocket.restype = c_int + ioctlsocket = external('ioctlsocket', + [socketfd_type, rffi.LONG, rffi.ULONGP], + rffi.INT) if _POSIX: poll = external('poll', [lltype.Ptr(pollfd), nfds_t, rffi.INT], rffi.INT) elif MS_WINDOWS: - XXX - select = socketdll.select - select.argtypes = [c_int, - POINTER(fd_set), POINTER(fd_set), POINTER(fd_set), - POINTER(timeval)] - select.restype = c_int - - WSACreateEvent = socketdll.WSACreateEvent - WSACreateEvent.argtypes = [] - WSACreateEvent.restype = WSAEVENT - - WSACloseEvent = socketdll.WSACloseEvent - WSACloseEvent.argtypes = [WSAEVENT] - WSACloseEvent.restype = c_int - - WSAEventSelect = socketdll.WSAEventSelect - WSAEventSelect.argtypes = [socketfd_type, WSAEVENT, rffi.LONG] - WSAEventSelect.restype = c_int - - WSAWaitForMultipleEvents = socketdll.WSAWaitForMultipleEvents - WSAWaitForMultipleEvents.argtypes = [rffi.LONG, POINTER(WSAEVENT), - c_int, rffi.LONG, c_int] - WSAWaitForMultipleEvents.restype = c_long - - WSAEnumNetworkEvents = socketdll.WSAEnumNetworkEvents - WSAEnumNetworkEvents.argtypes = [socketfd_type, WSAEVENT, - POINTER(WSANETWORKEVENTS)] - WSAEnumNetworkEvents.restype = c_int + select = external('select', + [rffi.INT, lltype.Ptr(fd_set), lltype.Ptr(fd_set), + lltype.Ptr(fd_set), lltype.Ptr(timeval)], + rffi.INT) + + ##WSACreateEvent = socketdll.WSACreateEvent + ##WSACreateEvent.argtypes = [] + ##WSACreateEvent.restype = WSAEVENT + + ##WSACloseEvent = socketdll.WSACloseEvent + ##WSACloseEvent.argtypes = [WSAEVENT] + ##WSACloseEvent.restype = c_int + + ##WSAEventSelect = socketdll.WSAEventSelect + ##WSAEventSelect.argtypes = [socketfd_type, WSAEVENT, rffi.LONG] + ##WSAEventSelect.restype = c_int + + ##WSAWaitForMultipleEvents = socketdll.WSAWaitForMultipleEvents + ##WSAWaitForMultipleEvents.argtypes = [rffi.LONG, POINTER(WSAEVENT), + ## c_int, rffi.LONG, c_int] + ##WSAWaitForMultipleEvents.restype = c_long + + ##WSAEnumNetworkEvents = socketdll.WSAEnumNetworkEvents + ##WSAEnumNetworkEvents.argtypes = [socketfd_type, WSAEVENT, + ## POINTER(WSANETWORKEVENTS)] + ##WSAEnumNetworkEvents.restype = c_int if MS_WINDOWS: WSAData = cConfig.WSAData - WSAStartup = socketdll.WSAStartup - WSAStartup.argtypes = [c_int, POINTER(WSAData)] - WSAStartup.restype = c_int - WSAStartup.libraries = ('ws2_32',) - - WSAGetLastError = socketdll.WSAGetLastError - WSAGetLastError.argtypes = [] - WSAGetLastError.restype = c_int + WSAStartup = external('WSAStartup', [rffi.INT, lltype.Ptr(WSAData)], + rffi.INT) + + WSAGetLastError = external('WSAGetLastError', [], rffi.INT) geterrno = WSAGetLastError import errno Modified: pypy/dist/pypy/rlib/getaddrinfo.py ============================================================================== --- pypy/dist/pypy/rlib/getaddrinfo.py (original) +++ pypy/dist/pypy/rlib/getaddrinfo.py Sat Sep 22 12:36:04 2007 @@ -3,10 +3,11 @@ This is a rewrite of the CPython source: Modules/getaddrinfo.c """ -from ctypes import POINTER, sizeof, cast, pointer -from pypy.rlib import _rsocket_ctypes as _c +from pypy.rlib import _rsocket_rffi as _c from pypy.rlib.rsocket import GAIError, CSocketError from pypy.rlib.rsocket import gethost_common, make_address +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.rarithmetic import r_uint # valid flags for addrinfo AI_MASK = (_c.AI_PASSIVE | _c.AI_CANONNAME | _c.AI_NUMERICHOST) @@ -43,7 +44,7 @@ if name == "": return False for c in name: - if c not in "012345789": + if not ("0" <= c <= "9"): return False return True @@ -109,12 +110,13 @@ sp = _c.getservbyname(servname, proto) if not sp: raise GAIError(EAI_SERVICE) - port = sp.contents.s_port + port = sp.c_s_port if socktype == GAI_ANY: - if sp.contents.s_proto == "udp": + s_proto = rffi.charp2str(sp.c_s_proto) + if s_proto == "udp": socktype = _c.SOCK_DGRAM protocol = _c.IPPROTO_UDP - elif sp.contents.s_proto == "tcp": + elif s_proto == "tcp": socktype = _c.SOCK_STREAM protocol = _c.IPPROTO_TCP else: @@ -127,17 +129,24 @@ result = [] if family in (_c.AF_UNSPEC, _c.AF_INET): - sin = _c.sockaddr_in(sin_family=_c.AF_INET, sin_port=port) - if flags & _c.AI_PASSIVE: - sin.sin_addr.s_addr = 0x0 # addrany - else: - sin.sin_addr.s_addr = 0x0100007f # loopback - - addr = make_address(cast(pointer(sin), POINTER(_c.sockaddr)), - sizeof(_c.sockaddr_in), address_to_fill) + sin = rffi.make(_c.sockaddr_in) + try: + rffi.setintfield(sin, 'c_sin_family', _c.AF_INET) + rffi.setintfield(sin, 'c_sin_port', port) + if flags & _c.AI_PASSIVE: + s_addr = 0x0 # addrany + else: + s_addr = 0x0100007f # loopback + rffi.setintfield(sin.c_sin_addr, 'c_s_addr', s_addr) - result.append((_c.AF_INET, socktype, protocol, "", # xxx canonname meaningless? "anyaddr" - addr)) + addr = make_address(rffi.cast(_c.sockaddr_ptr, sin), + rffi.sizeof(_c.sockaddr_in), + address_to_fill) + + result.append((_c.AF_INET, socktype, protocol, "", # xxx canonname meaningless? "anyaddr" + addr)) + finally: + lltype.free(sin, flavor='raw') if not result: raise GAIError(EAI_FAMILY) @@ -147,38 +156,39 @@ if family in (_c.AF_UNSPEC, _c.AF_INET): packedaddr = _c.inet_addr(hostname) - if packedaddr != INADDR_NONE: + if packedaddr != rffi.cast(rffi.UINT, INADDR_NONE): - v4a = _c.ntohl(packedaddr) - if (v4a & 0xf0000000 == 0xe0000000 or # IN_MULTICAST() - v4a & 0xe0000000 == 0xe0000000): # IN_EXPERIMENTAL() + v4a = rffi.cast(lltype.Unsigned, _c.ntohl(packedaddr)) + if (v4a & r_uint(0xf0000000) == r_uint(0xe0000000) or # IN_MULTICAST() + v4a & r_uint(0xe0000000) == r_uint(0xe0000000)): # IN_EXPERIMENTAL() flags &= ~_c.AI_CANONNAME v4a >>= 24 # = IN_CLASSA_NSHIFT - if v4a in (0, 127): # = IN_LOOPBACKNET + if v4a == r_uint(0) or v4a == r_uint(127): # = IN_LOOPBACKNET flags &= ~_c.AI_CANONNAME + sin = rffi.make(_c.sockaddr_in) + try: + rffi.setintfield(sin, 'c_sin_family', _c.AF_INET) + rffi.setintfield(sin, 'c_sin_port', port) + rffi.setintfield(sin.c_sin_addr, 'c_s_addr', packedaddr) + addr = make_address(rffi.cast(_c.sockaddr_ptr, sin), + sizeof(_c.sockaddr_in), + address_to_fill) + finally: + lltype.free(sin, flavor='raw') + if not flags & _c.AI_CANONNAME: - sin = _c.sockaddr_in(sin_family=_c.AF_INET, sin_port=port) - sin.sin_addr.s_addr = packedaddr - addr = make_address(cast(pointer(sin), POINTER(_c.sockaddr)), - sizeof(_c.sockaddr_in), address_to_fill) - return [(_c.AF_INET, socktype, protocol, None, addr)] + canonname = "" else: - sin = _c.sockaddr_in(sin_family=_c.AF_INET, sin_port=port) - - sin.sin_addr.s_addr = packedaddr - # getaddrinfo() is a name->address translation function, - # and it looks strange that we do addr->name translation here. + # and it looks strange that we do addr->name translation + # here. # This is what python2.3 did on Windows: # if sys.version < (2, 4): # canonname = get_name(hostname, sin.sin_addr, # sizeof(_c.in_addr)) canonname = hostname - - addr = make_address(cast(pointer(sin), POINTER(_c.sockaddr)), - sizeof(_c.sockaddr_in), address_to_fill) - return [(_c.AF_INET, socktype, protocol, canonname, addr)] + return [(_c.AF_INET, socktype, protocol, canonname, addr)] if flags & _c.AI_NUMERICHOST: raise GAIError(EAI_NONAME) @@ -191,19 +201,19 @@ raise GAIError(EAI_FAIL) -def get_name(hostname, addr, addrlen): - hostent = _c.gethostbyaddr(pointer(addr), addrlen, _c.AF_INET) - - # if reverse lookup fail, - # return address anyway to pacify calling application. - if not hostent: - return hostname - - hname, aliases, address_list = gethost_common("", hostent) - if hostent and hostent.contents.h_name and hostent.contents.h_addr_list[0]: - return hostent.contents.h_name - else: - return hostname +##def get_name(hostname, addr, addrlen): +## hostent = _c.gethostbyaddr(pointer(addr), addrlen, _c.AF_INET) +## +## # if reverse lookup fail, +## # return address anyway to pacify calling application. +## if not hostent: +## return hostname +## +## hname, aliases, address_list = gethost_common("", hostent) +## if hostent and hostent.contents.h_name and hostent.contents.h_addr_list[0]: +## return hostent.contents.h_name +## else: +## return hostname def get_addr(hostname, socktype, protocol, port, address_to_fill): hostent = _c.gethostbyname(hostname) @@ -216,11 +226,14 @@ result = [] for address in address_list: - if address.addr.sa_family == _c.AF_INET: - a = cast(pointer(address.addr), POINTER(_c.sockaddr_in)).contents - a.sin_port = port & 0xffff - addr = make_address(pointer(address.addr),address.addrlen,address_to_fill) - result.append((address.addr.sa_family, + if address.family == _c.AF_INET: + a = address.lock(_c.sockaddr_in) + rffi.setintfield(a, 'c_sin_port', port & 0xffff) + address.unlock() + a = address.lock() + addr = make_address(a, address.addrlen, address_to_fill) + address.unlock() + result.append((address.family, socktype, protocol, "", # XXX canonname? Modified: pypy/dist/pypy/rlib/getnameinfo.py ============================================================================== --- pypy/dist/pypy/rlib/getnameinfo.py (original) +++ pypy/dist/pypy/rlib/getnameinfo.py Sat Sep 22 12:36:04 2007 @@ -2,9 +2,10 @@ An RPython implementation of getnameinfo() based on ctypes. This is a rewrite of the CPython source: Modules/getaddrinfo.c """ -from ctypes import POINTER, sizeof, cast, pointer -from pypy.rlib import _rsocket_ctypes as _c +from pypy.rlib import _rsocket_rffi as _c from pypy.rlib.rsocket import RSocketError, GAIError +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.rarithmetic import r_uint NI_NOFQDN = 0x00000001 NI_NUMERICHOST = 0x00000002 @@ -14,55 +15,55 @@ def _getservicename(sin_port, flags): if flags & NI_NUMERICSERV: - sp = None + sp = lltype.nullptr(_c.cConfig.servent) elif flags & NI_DGRAM: sp = _c.getservbyport(sin_port, "udp") else: sp = _c.getservbyport(sin_port, "tcp") if sp: - serv = sp.contents.s_name + serv = rffi.charp2str(sp.c_s_name) else: serv = "%d" % _c.ntohs(sin_port) return serv -def getnameinfo(_addr, flags): - addr = _addr.addr - - if addr.sa_family != _c.AF_INET: +def getnameinfo(address, flags): + if address.family != _c.AF_INET: raise RSocketError("unknown address family") - sockaddr = cast(pointer(addr), POINTER(_c.sockaddr_in)).contents - sin_port = sockaddr.sin_port - sin_addr = sockaddr.sin_addr - - v4a = _c.ntohl(sin_addr.s_addr) - if (v4a & 0xf0000000 == 0xe0000000 or # IN_MULTICAST() - v4a & 0xe0000000 == 0xe0000000): # IN_EXPERIMENTAL() - flags |= NI_NUMERICHOST - # XXX Why does this work in CPython? - # v4a >>= 24 # = IN_CLASSA_NSHIFT - # if v4a in (0, 127): # = IN_LOOPBACKNET - # flags |= NI_NUMERICHOST - numsize = _c.INET_ADDRSTRLEN - - serv = _getservicename(sin_port, flags) + sockaddr = address.lock(_c.sockaddr_in) + try: + sin_port = sockaddr.c_sin_port + sin_addr = sockaddr.c_sin_addr + + v4a = rffi.cast(lltype.Unsigned, _c.ntohl(sin_addr.c_s_addr)) + if (v4a & r_uint(0xf0000000) == r_uint(0xe0000000) or # IN_MULTICAST() + v4a & r_uint(0xe0000000) == r_uint(0xe0000000)): # IN_EXPERIMENTAL() + flags |= NI_NUMERICHOST + # XXX Why does this work in CPython? + # v4a >>= 24 # = IN_CLASSA_NSHIFT + # if v4a in (0, 127): # = IN_LOOPBACKNET + # flags |= NI_NUMERICHOST + numsize = _c.INET_ADDRSTRLEN + + serv = _getservicename(sin_port, flags) + + if not (flags & NI_NUMERICHOST): + hostent = _c.gethostbyaddr(sin_addr, sizeof(_c.in_addr), + sockaddr.c_sin_family) + else: + hostent = None + + if hostent: + from pypy.rlib.rsocket import gethost_common + host, _, _ = gethost_common("", hostent) + else: + host = rffi.charp2str(_c.inet_ntoa(sin_addr)) - if not (flags & NI_NUMERICHOST): - hostent = _c.gethostbyaddr(pointer(sin_addr), sizeof(_c.in_addr), addr.sa_family) - else: - hostent = None + finally: + address.unlock() - if hostent: - from pypy.rlib.rsocket import gethost_common - host, _, _ = gethost_common("", hostent) - else: - from pypy.rlib.rsocket import copy_buffer - host = _c.inet_ntoa(sin_addr) - #buf = copy_buffer(ptr, len(ptr)) - #host = buf.raw - return host, serv Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Sat Sep 22 12:36:04 2007 @@ -29,8 +29,9 @@ if _c.MS_WINDOWS: def rsocket_startup(): - wsadata = _c.WSAData() - res = _c.WSAStartup(1, byref(wsadata)) + wsadata = lltype.malloc(_c.WSAData, flavor='raw', zero=True) + res = _c.WSAStartup(1, wsadata) + lltype.free(wsadata, flavor='raw') assert res == 0 else: def rsocket_startup(): @@ -535,8 +536,10 @@ _c.fcntl(self.fd, _c.F_SETFL, delay_flag) elif hasattr(_c, 'ioctlsocket'): def _setblocking(self, block): - flag = c_ulong(not block) - _c.ioctlsocket(self.fd, _c.FIONBIO, byref(flag)) + flag = lltype.malloc(rffi.ULONGP.TO, 1, flavor='raw') + flag[0] = rffi.cast(rffi.ULONG, not block) + _c.ioctlsocket(self.fd, _c.FIONBIO, flag) + lltype.free(flag, flavor='raw') if hasattr(_c, 'poll'): def _select(self, for_writing): @@ -566,19 +569,23 @@ def _select(self, for_writing): """Returns 0 when reading/writing is possible, 1 when timing out and -1 on error.""" - XXX if self.timeout <= 0.0 or self.fd < 0: # blocking I/O or no socket. return 0 - tv = _c.timeval(tv_sec=int(self.timeout), - tv_usec=int((self.timeout-int(self.timeout)) - * 1000000)) - fds = _c.fd_set(fd_count=1) - fds.fd_array[0] = self.fd + tv = rffi.make(_c.timeval) + rffi.setintfield(tv, 'tv_sec', int(self.timeout)) + rffi.setintfield(tv, 'tv_usec', int((self.timeout-int(self.timeout)) + * 1000000)) + fds = rffi.make(_c.fd_set) + rffi.setintfield(fds, 'fd_count', 1) + fds.fd_array[0] = rffi.cast(socketfd_type, self.fd) + null = lltype.nullptr(fd_set) if for_writing: - n = _c.select(self.fd + 1, None, byref(fds), None, byref(tv)) + n = _c.select(self.fd + 1, null, fds, null, tv) else: - n = _c.select(self.fd + 1, byref(fds), None, None, byref(tv)) + n = _c.select(self.fd + 1, fds, null, null, tv) + lltype.free(fds, flavor='raw') + lltype.free(tv, flavor='raw') if n < 0: return -1 if n == 0: @@ -955,6 +962,7 @@ class GAIError(SocketErrorWithErrno): applevelerrcls = 'gaierror' def get_msg(self): + # this method may be patched below return rffi.charp2str(_c.gai_strerror(self.errno)) class HSocketError(SocketError): @@ -1253,12 +1261,10 @@ # if not getattr(_c, 'getaddrinfo', None): - XXX from pypy.rlib.getaddrinfo import getaddrinfo from pypy.rlib.getaddrinfo import GAIError_getmsg GAIError.get_msg = GAIError_getmsg if not getattr(_c, 'getnameinfo', None): - XXX from pypy.rlib.getnameinfo import getnameinfo from pypy.rlib.getnameinfo import NI_NUMERICHOST, NI_NUMERICSERV Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Sat Sep 22 12:36:04 2007 @@ -48,7 +48,8 @@ allnames = [name] + aliases for n in allnames: assert isinstance(n, str) - assert 'localhost' in allnames + if sys.platform != 'win32': + assert 'localhost' in allnames for a in address_list: if isinstance(a, INETAddress) and a.get_host() == "127.0.0.1": break # ok @@ -114,10 +115,17 @@ assert addr.eq(sock.getsockname()) sock.listen(1) s2 = RSocket(AF_INET, SOCK_STREAM) - thread.start_new_thread(s2.connect, (addr,)) + def connecting(): + s2.connect(addr) + lock.release() + lock = thread.allocate_lock() + lock.acquire() + thread.start_new_thread(connecting, ()) print 'waiting for connexion' s1, addr2 = sock.accept() print 'connexion accepted' + lock.acquire() + print 'connecting side knows that the connexion was accepted too' assert addr.eq(s2.getpeername()) assert addr2.eq(s2.getsockname()) assert addr2.eq(s1.getpeername()) Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Sat Sep 22 12:36:04 2007 @@ -471,7 +471,7 @@ # ____________________________________________ -def get_ctypes_callable(funcptr): +def get_ctypes_callable(funcptr, calling_conv): if getattr(funcptr._obj, 'sources', None): # give up - for tests with an inlined bit of C code raise NotImplementedError("cannot call a C function defined in " @@ -491,7 +491,8 @@ if not libpath and os.path.isabs(libname): libpath = libname if libpath: - clib = ctypes.cdll.LoadLibrary(libpath) + dllclass = getattr(ctypes, calling_conv + 'dll') + clib = dllclass.LoadLibrary(libpath) cfunc = getattr(clib, funcname, None) if cfunc is not None: break @@ -518,15 +519,16 @@ class LL2CtypesCallable(object): # a special '_callable' object that invokes ctypes - def __init__(self, FUNCTYPE): + def __init__(self, FUNCTYPE, calling_conv): self.FUNCTYPE = FUNCTYPE + self.calling_conv = calling_conv self.trampoline = None #self.funcptr = ... set later def __call__(self, *argvalues): if self.trampoline is None: # lazily build the corresponding ctypes function object - cfunc = get_ctypes_callable(self.funcptr) + cfunc = get_ctypes_callable(self.funcptr, self.calling_conv) self.trampoline = get_ctypes_trampoline(self.FUNCTYPE, cfunc) # perform the call return self.trampoline(*argvalues) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sat Sep 22 12:36:04 2007 @@ -28,7 +28,7 @@ def llexternal(name, args, result, _callable=None, sources=[], includes=[], libraries=[], include_dirs=[], sandboxsafe=False, - canraise=False, _nowrapper=False): + canraise=False, _nowrapper=False, calling_conv='c'): """Build an external function that will invoke the C function 'name' with the given 'args' types and 'result' type. @@ -40,7 +40,7 @@ """ ext_type = lltype.FuncType(args, result) if _callable is None: - _callable = ll2ctypes.LL2CtypesCallable(ext_type) + _callable = ll2ctypes.LL2CtypesCallable(ext_type, calling_conv) funcptr = lltype.functionptr(ext_type, name, external='C', sources=tuple(sources), includes=tuple(includes), From arigo at codespeak.net Sat Sep 22 12:43:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 12:43:20 +0200 (CEST) Subject: [pypy-svn] r46815 - pypy/dist/pypy/rpython Message-ID: <20070922104320.4A42E8110@code0.codespeak.net> Author: arigo Date: Sat Sep 22 12:43:19 2007 New Revision: 46815 Modified: pypy/dist/pypy/rpython/rint.py Log: Oups! I fixed a typo that wasn't a typo at all. Revert and add a comment. Modified: pypy/dist/pypy/rpython/rint.py ============================================================================== --- pypy/dist/pypy/rpython/rint.py (original) +++ pypy/dist/pypy/rpython/rint.py Sat Sep 22 12:43:19 2007 @@ -344,17 +344,17 @@ self = self.as_int vlist = hop.inputargs(self) if hop.s_result.unsigned: + # implement '-r_uint(x)' with unsigned subtraction '0 - x' zero = self.lowleveltype._defl() vlist.insert(0, hop.inputconst(self.lowleveltype, zero)) - return hop.genop(self.opprefix + 'neg', vlist, resulttype=self) + return hop.genop(self.opprefix + 'sub', vlist, resulttype=self) else: return hop.genop(self.opprefix + 'neg', vlist, resulttype=self) def rtype_neg_ovf(self, hop): self = self.as_int if hop.s_result.unsigned: - # this is supported (and turns into just uint_neg) for - # rbigint.py + # this is supported (and turns into just 0-x) for rbigint.py hop.exception_cannot_occur() return self.rtype_neg(hop) else: From cfbolz at codespeak.net Sat Sep 22 13:47:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 22 Sep 2007 13:47:53 +0200 (CEST) Subject: [pypy-svn] r46816 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070922114753.E039280F6@code0.codespeak.net> Author: cfbolz Date: Sat Sep 22 13:47:52 2007 New Revision: 46816 Added: pypy/extradoc/sprintinfo/gothenburg-2007/ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: start the gtbg announcement, paste in tasks Added: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 22 13:47:52 2007 @@ -0,0 +1,83 @@ +PyPy G?teborg Cleanup Sprint November 19 - November 25 2007 +=========================================================== + +XXX + +Topics and goals +---------------- + +Meta-Tasks +++++++++++ + + - Tracker-gardening + - Start slowly thinking about 1.1 plans + +Translation toolchain ++++++++++++++++++++++ + + + - finish rctypes removal + - implement correct implicit GIL release + + - move away from implicit keepalives + - raw_malloc should be split, rffi one should not use obmalloc (it's + not thread-safe) + - kill simulator, run the semi space tests on llinterp (may happen + before) + - have a moving gc correct version of the gc framework transformation + - delegate finding type stuff like vtables etc to GC, cleaner interface + for rtti, simplify translator/c/gc.py + - think about approaches to id, especially concerning boehm, where the + id will keep the object alive and concerning a moving GC + + - clean up the tangle of including headers in the C backend + + - review pdbplus, especially the graph commands, also in the light of + https://codespeak.net/issue/pypy-dev/issue303 and the fact that we + can have more than one translator/annotator around (with the + timeshifter) + + - Kill half concrete wrapper + + - Kill opaque hacks on the C backend + + - decide how to implement constants in rffi + + - think about/remove orphaned parts + + - finish ll2ctypes, especially rffi_platform + +Interpreter ++++++++++++ + + - there's a shadow tracking bug it seems + + - fix taint space + + - review the things implemented at applevel whether they are + performance-critical + +JIT ++++ + + - start writing real unit tests for the merging logic + + +Location +-------- + + + +Registration +------------ + +If you'd like to come, please subscribe to the `pypy-sprint mailing list`_ +and drop a note about your interests and post any questions. More +organisational information will be send to that list. We'll keep a list +of `people`_ which we'll update (which you can do so yourself if you +have codespeak commit rights). If you have no clue where to stay in +Bern ask around on the list, maybe we can help you find something or +find some other solution. + +.. _`pypy-sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint +.. _`people`: http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2007/people.html Added: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sat Sep 22 13:47:52 2007 @@ -0,0 +1,50 @@ +People coming to the Gothenburg sprint November 2007 +==================================================== + +People who have a ``?`` in their arrive/depart or accomodation +column are known to be coming but there are no details +available yet from them. + +==================== ============== ===================== + Name Arrive/Depart Accomodation +==================== ============== ===================== +Carl Friedrich Bolz ? ? +==================== ============== ===================== + + +People on the following list were present at previous sprints: + +==================== ============== ===================== + Name Arrive/Depart Accomodation +==================== ============== ===================== +Armin Rigo ? ? +Michael Hudson ? ? +Antonio Cuni ? ? +Samuele Pedroni ? ? +Anders Chrigstroem ? ? +Maciej Fijalkowski ? ? +Andrew Thompson ? ? +Stephan Diehl ? ? +Eric van Riet paap ? ? +Niko Matsakis ? ? +Leonardo Santagada ? ? +Richard Emslie ? ? +Christian Tismer ? ? +Alexander Schremmer ? ? +Jacob Hallen ? ? +Aurelien Campeas ? ? +Alexandre Fayolle ? ? +Lene Wagner ? ? +Amaury Forgeot d'Arc ? ? +Valentino Volonghi ? ? +Boris Feigin ? ? +Bert Freudenberg ? ? +Laura Creighton ? ? +Beatrice Duering ? ? +Johan Hahn ? ? +Holger Krekel ? ? +Anders Chrigstroem ? ? +Samuele Pedroni ? ? +Anders Lehmann ? ? +Niklaus Haldimann ? ? +==================== ============== ===================== From arigo at codespeak.net Sat Sep 22 13:58:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 13:58:18 +0200 (CEST) Subject: [pypy-svn] r46817 - pypy/dist/pypy/rlib Message-ID: <20070922115818.C2CC38106@code0.codespeak.net> Author: arigo Date: Sat Sep 22 13:58:18 2007 New Revision: 46817 Modified: pypy/dist/pypy/rlib/rsocket.py Log: Minor fix. Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Sat Sep 22 13:58:18 2007 @@ -545,7 +545,7 @@ def _select(self, for_writing): """Returns 0 when reading/writing is possible, 1 when timing out and -1 on error.""" - if self.timeout <= 0.0 or self.fd < 0: + if self.timeout <= 0.0 or self.fd == _c.INVALID_SOCKET: # blocking I/O or no socket. return 0 pollfd = rffi.make(_c.pollfd) @@ -569,7 +569,7 @@ def _select(self, for_writing): """Returns 0 when reading/writing is possible, 1 when timing out and -1 on error.""" - if self.timeout <= 0.0 or self.fd < 0: + if self.timeout <= 0.0 or self.fd == _c.INVALID_SOCKET: # blocking I/O or no socket. return 0 tv = rffi.make(_c.timeval) From arigo at codespeak.net Sat Sep 22 13:58:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 13:58:39 +0200 (CEST) Subject: [pypy-svn] r46818 - in pypy/dist/pypy/rlib: . test Message-ID: <20070922115839.2A0418106@code0.codespeak.net> Author: arigo Date: Sat Sep 22 13:58:38 2007 New Revision: 46818 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/getaddrinfo.py pypy/dist/pypy/rlib/test/test_rsocket.py Log: More progress on Windows. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Sat Sep 22 13:58:38 2007 @@ -441,7 +441,7 @@ inet_addr = external('inet_addr', [rffi.CCHARP], rffi.UINT) socklen_t_ptr = lltype.Ptr(rffi.CFixedArray(socklen_t, 1)) socketaccept = external('accept', [socketfd_type, sockaddr_ptr, - socklen_t_ptr], rffi.INT) + socklen_t_ptr], socketfd_type) socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t], rffi.INT) socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT) Modified: pypy/dist/pypy/rlib/getaddrinfo.py ============================================================================== --- pypy/dist/pypy/rlib/getaddrinfo.py (original) +++ pypy/dist/pypy/rlib/getaddrinfo.py Sat Sep 22 13:58:38 2007 @@ -172,7 +172,7 @@ rffi.setintfield(sin, 'c_sin_port', port) rffi.setintfield(sin.c_sin_addr, 'c_s_addr', packedaddr) addr = make_address(rffi.cast(_c.sockaddr_ptr, sin), - sizeof(_c.sockaddr_in), + rffi.sizeof(_c.sockaddr_in), address_to_fill) finally: lltype.free(sin, flavor='raw') Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Sat Sep 22 13:58:38 2007 @@ -2,6 +2,11 @@ from pypy.rlib import rsocket from pypy.rlib.rsocket import * +# cannot test error codes in Win32 because ll2ctypes doesn't save +# the errors that WSAGetLastError() should return, making it likely +# that other operations stamped on it inbetween. +errcodesok = sys.platform != 'win32' + def setup_module(mod): rsocket_startup() @@ -198,12 +203,14 @@ assert addr.eq(sock.getsockname()) sock.listen(1) err = py.test.raises(CSocketError, sock.accept) - assert err.value.errno in (errno.EAGAIN, errno.EWOULDBLOCK) + if errcodesok: + assert err.value.errno in (errno.EAGAIN, errno.EWOULDBLOCK) s2 = RSocket(AF_INET, SOCK_STREAM) s2.setblocking(False) err = py.test.raises(CSocketError, s2.connect, addr) - assert err.value.errno in (errno.EINPROGRESS, errno.EWOULDBLOCK) + if errcodesok: + assert err.value.errno in (errno.EINPROGRESS, errno.EWOULDBLOCK) s1, addr2 = sock.accept() s1.setblocking(False) @@ -212,7 +219,8 @@ assert addr2.eq(s1.getpeername()) err = s2.connect_ex(addr) # should now work - assert err in (0, errno.EISCONN) + if errcodesok: + assert err in (0, errno.EISCONN) s1.send('?') import time @@ -220,7 +228,8 @@ buf = s2.recv(100) assert buf == '?' err = py.test.raises(CSocketError, s1.recv, 5000) - assert err.value.errno in (errno.EAGAIN, errno.EWOULDBLOCK) + if errcodesok: + assert err.value.errno in (errno.EAGAIN, errno.EWOULDBLOCK) count = s2.send('x'*50000) assert 1 <= count <= 50000 while count: # Recv may return less than requested @@ -270,7 +279,8 @@ def test_connect_ex(): s = RSocket() err = s.connect_ex(INETAddress('0.0.0.0', 0)) # should not work - assert err in (errno.ECONNREFUSED, errno.EADDRNOTAVAIL) + if errcodesok: + assert err in (errno.ECONNREFUSED, errno.EADDRNOTAVAIL) def test_getsetsockopt(): From arigo at codespeak.net Sat Sep 22 14:05:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 14:05:04 +0200 (CEST) Subject: [pypy-svn] r46819 - in pypy/dist/pypy/rlib: . test Message-ID: <20070922120504.4D0AA8110@code0.codespeak.net> Author: arigo Date: Sat Sep 22 14:05:02 2007 New Revision: 46819 Modified: pypy/dist/pypy/rlib/rsocket.py pypy/dist/pypy/rlib/test/test_rsocket.py Log: test_rsocket passes on Windows. Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Sat Sep 22 14:05:02 2007 @@ -569,17 +569,18 @@ def _select(self, for_writing): """Returns 0 when reading/writing is possible, 1 when timing out and -1 on error.""" - if self.timeout <= 0.0 or self.fd == _c.INVALID_SOCKET: + timeout = self.timeout + if timeout <= 0.0 or self.fd == _c.INVALID_SOCKET: # blocking I/O or no socket. return 0 tv = rffi.make(_c.timeval) - rffi.setintfield(tv, 'tv_sec', int(self.timeout)) - rffi.setintfield(tv, 'tv_usec', int((self.timeout-int(self.timeout)) - * 1000000)) + rffi.setintfield(tv, 'c_tv_sec', int(timeout)) + rffi.setintfield(tv, 'c_tv_usec', int((timeout-int(timeout)) + * 1000000)) fds = rffi.make(_c.fd_set) - rffi.setintfield(fds, 'fd_count', 1) - fds.fd_array[0] = rffi.cast(socketfd_type, self.fd) - null = lltype.nullptr(fd_set) + rffi.setintfield(fds, 'c_fd_count', 1) + fds.c_fd_array[0] = rffi.cast(_c.socketfd_type, self.fd) + null = lltype.nullptr(_c.fd_set) if for_writing: n = _c.select(self.fd + 1, null, fds, null, tv) else: Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Sat Sep 22 14:05:02 2007 @@ -334,10 +334,14 @@ assert inet_ntoa('\x01\x02\x03\x04') == '1.2.3.4' def test_inet_pton(): + if not hasattr(rsocket, 'inet_pton'): + py.test.skip("no inet_pton()") assert inet_pton(AF_INET, '1.2.3.5') == '\x01\x02\x03\x05' py.test.raises(SocketError, inet_pton, AF_INET, '127.0.0.256') def test_inet_ntop(): + if not hasattr(rsocket, 'inet_ntop'): + py.test.skip("no inet_ntop()") assert inet_ntop(AF_INET, '\x01\x02\x03\x05') == '1.2.3.5' class TestTCP: From arigo at codespeak.net Sat Sep 22 14:09:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 14:09:48 +0200 (CEST) Subject: [pypy-svn] r46820 - pypy/dist/pypy/rlib Message-ID: <20070922120948.8B5988110@code0.codespeak.net> Author: arigo Date: Sat Sep 22 14:09:47 2007 New Revision: 46820 Modified: pypy/dist/pypy/rlib/getnameinfo.py Log: Fix Modified: pypy/dist/pypy/rlib/getnameinfo.py ============================================================================== --- pypy/dist/pypy/rlib/getnameinfo.py (original) +++ pypy/dist/pypy/rlib/getnameinfo.py Sat Sep 22 14:09:47 2007 @@ -51,7 +51,8 @@ serv = _getservicename(sin_port, flags) if not (flags & NI_NUMERICHOST): - hostent = _c.gethostbyaddr(sin_addr, sizeof(_c.in_addr), + p = rffi.cast(rffi.VOIDP, sin_addr) + hostent = _c.gethostbyaddr(p, rffi.sizeof(_c.in_addr), sockaddr.c_sin_family) else: hostent = None From arigo at codespeak.net Sat Sep 22 14:24:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 14:24:13 +0200 (CEST) Subject: [pypy-svn] r46821 - in pypy/dist/pypy: rpython/memory/gctransform translator/c/test Message-ID: <20070922122413.55D1E810E@code0.codespeak.net> Author: arigo Date: Sat Sep 22 14:24:12 2007 New Revision: 46821 Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/translator/c/test/test_lltyped.py Log: I needed raw zero mallocs of an array, after all. What's left out now is only the same thing when, in addition, the array is a normal lltype array with a length. This thing would be a bit strange because it's normally not needed to access external C functions. Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Sat Sep 22 14:24:12 2007 @@ -375,13 +375,18 @@ raise MemoryError() return result mh._ll_malloc_fixedsize = _ll_malloc_fixedsize - - def _ll_malloc_varsize_no_length(length, size, itemsize): + + def _ll_compute_size(length, size, itemsize): try: varsize = ovfcheck(itemsize * length) tot_size = ovfcheck(size + varsize) except OverflowError: raise MemoryError() + return tot_size + _ll_compute_size._inline_ = True + + def _ll_malloc_varsize_no_length(length, size, itemsize): + tot_size = _ll_compute_size(length, size, itemsize) result = mh.allocate(tot_size) if not result: raise MemoryError() @@ -395,6 +400,15 @@ return result mh.ll_malloc_varsize = ll_malloc_varsize + def _ll_malloc_varsize_no_length_zero(length, size, itemsize): + tot_size = _ll_compute_size(length, size, itemsize) + result = mh.allocate(tot_size) + if not result: + raise MemoryError() + lladdress.raw_memclear(result, tot_size) + return result + mh.ll_malloc_varsize_no_length_zero = _ll_malloc_varsize_no_length_zero + return mh class GCTransformer(BaseGCTransformer): @@ -407,6 +421,7 @@ ll_raw_malloc_fixedsize = mh._ll_malloc_fixedsize ll_raw_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length ll_raw_malloc_varsize = mh.ll_malloc_varsize + ll_raw_malloc_varsize_no_length_zero = mh.ll_malloc_varsize_no_length_zero stack_mh = mallocHelpers() stack_mh.allocate = lambda size: llop.stack_malloc(llmemory.Address, size) @@ -419,6 +434,8 @@ ll_raw_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address, inline=False) self.raw_malloc_varsize_ptr = self.inittime_helper( ll_raw_malloc_varsize, [lltype.Signed]*4, llmemory.Address, inline=False) + self.raw_malloc_varsize_no_length_zero_ptr = self.inittime_helper( + ll_raw_malloc_varsize_no_length_zero, [lltype.Signed]*3, llmemory.Address, inline=False) self.stack_malloc_fixedsize_ptr = self.inittime_helper( ll_stack_malloc_fixedsize, [lltype.Signed], llmemory.Address) @@ -495,17 +512,20 @@ def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size, c_offset_to_length): if c_offset_to_length is None: + if flags.get('zero'): + fnptr = self.raw_malloc_varsize_no_length_zero_ptr + else: + fnptr = self.raw_malloc_varsize_no_length_ptr v_raw = hop.genop("direct_call", - [self.raw_malloc_varsize_no_length_ptr, v_length, - c_const_size, c_item_size], + [fnptr, v_length, c_const_size, c_item_size], resulttype=llmemory.Address) else: + if flags.get('zero'): + raise NotImplementedError("raw zero varsize malloc with length field") v_raw = hop.genop("direct_call", [self.raw_malloc_varsize_ptr, v_length, c_const_size, c_item_size, c_offset_to_length], resulttype=llmemory.Address) - if flags.get('zero'): - raise NotImplementedError("raw zero varsize malloc") return v_raw def gct_free(self, hop): Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Sat Sep 22 14:24:12 2007 @@ -432,6 +432,27 @@ res = fn(100) assert res == 42 + def test_zero_raw_malloc_varsize(self): + # we don't support at the moment raw+zero mallocs with a length + # field to initialize + S = Struct('S', ('x', Signed), ('y', Array(Signed, hints={'nolength': True}))) + def f(n): + for length in range(n-1, -1, -1): + p = malloc(S, length, flavor='raw', zero=True) + if p.x != 0: + return -1 + p.x = n + for j in range(length): + if p.y[j] != 0: + return -3 + p.y[j] = n^j + free(p, flavor='raw') + return 42 + + fn = self.getcompiled(f, [int]) + res = fn(100) + assert res == 42 + def test_arithmetic_cornercases(self): import operator, sys from pypy.rlib.unroll import unrolling_iterable From arigo at codespeak.net Sat Sep 22 17:53:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 17:53:14 +0200 (CEST) Subject: [pypy-svn] r46827 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070922155314.661BE8156@code0.codespeak.net> Author: arigo Date: Sat Sep 22 17:53:14 2007 New Revision: 46827 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: ll2ctypes support for arrays of structures. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Sat Sep 22 17:53:14 2007 @@ -118,7 +118,10 @@ else: items = self._indexable(index) cobj = items[index] - return ctypes2lltype(ITEM, cobj) + if isinstance(ITEM, lltype.ContainerType): + return ctypes2lltype(lltype.Ptr(ITEM), ctypes.pointer(cobj)) + else: + return ctypes2lltype(ITEM, cobj) def _setitem(self, index, value, boundscheck=True): if boundscheck: @@ -220,10 +223,16 @@ cls = get_ctypes_type(ARRAY) carray = cls._malloc(container.getlength()) add_storage(container, _array_mixin, carray) - for i in range(container.getlength()): - item_value = container.items[i] # fish fish - carray.items[i] = lltype2ctypes(item_value) - remove_regular_array_content(container) + if not isinstance(ARRAY.OF, lltype.ContainerType): + for i in range(container.getlength()): + item_value = container.items[i] # fish fish + carray.items[i] = lltype2ctypes(item_value) + remove_regular_array_content(container) + else: + assert isinstance(ARRAY.OF, lltype.Struct) + for i in range(container.getlength()): + item_ptr = container.items[i] # fish fish + convert_struct(item_ptr, carray.items[i]) def remove_regular_array_content(container): for i in range(container.getlength()): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sat Sep 22 17:53:14 2007 @@ -517,6 +517,31 @@ lltype.free(a2, flavor='raw') lltype.free(s2, flavor='raw') + def test_arrayofstruct(self): + S1 = lltype.Struct('S1', ('x', lltype.Signed)) + A = lltype.Array(S1, hints={'nolength': True}) + a = lltype.malloc(A, 5, flavor='raw') + a[0].x = 100 + a[1].x = 101 + a[2].x = 102 + a[3].x = 103 + a[4].x = 104 + ac = lltype2ctypes(a, normalize=False) + assert ac.contents.items[0].x == 100 + assert ac.contents.items[2].x == 102 + ac.contents.items[3].x += 500 + assert a[3].x == 603 + a[4].x += 600 + assert ac.contents.items[4].x == 704 + a1 = ctypes2lltype(lltype.Ptr(A), ac) + assert a1 == a + assert a1[2].x == 102 + aitem1 = ctypes2lltype(lltype.Ptr(S1), + ctypes.pointer(ac.contents.items[1])) + assert aitem1.x == 101 + assert aitem1 == a1[1] + lltype.free(a, flavor='raw') + def test_get_errno(self): if sys.platform.startswith('win'): underscore_on_windows = '_' From arigo at codespeak.net Sat Sep 22 17:55:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 17:55:23 +0200 (CEST) Subject: [pypy-svn] r46828 - in pypy/dist/pypy/rlib: . test Message-ID: <20070922155523.94C2E814D@code0.codespeak.net> Author: arigo Date: Sat Sep 22 17:55:23 2007 New Revision: 46828 Added: pypy/dist/pypy/rlib/rpoll.py pypy/dist/pypy/rlib/test/test_rpoll.py Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket.py Log: Writing pypy.rlib.rpoll. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Sat Sep 22 17:55:23 2007 @@ -482,7 +482,9 @@ rffi.INT) if _POSIX: - poll = external('poll', [lltype.Ptr(pollfd), nfds_t, rffi.INT], rffi.INT) + pollfdarray = rffi.CArray(pollfd) + poll = external('poll', [lltype.Ptr(pollfdarray), nfds_t, rffi.INT], + rffi.INT) elif MS_WINDOWS: select = external('select', [rffi.INT, lltype.Ptr(fd_set), lltype.Ptr(fd_set), Added: pypy/dist/pypy/rlib/rpoll.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/rpoll.py Sat Sep 22 17:55:23 2007 @@ -0,0 +1,61 @@ +""" +An RPython implementation of select.poll() based on rffi. +Note that this is not a drop-in replacement: the interface is +simplified - instead of a polling object there is only a poll() +function that directly takes a dictionary as argument. +""" + +import os +from pypy.rlib import _rsocket_rffi as _c +from pypy.rpython.lltypesystem import lltype, rffi + +# ____________________________________________________________ +# events +# +eventnames = '''POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL + POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG'''.split() + +eventnames = [name for name in eventnames + if getattr(_c.cConfig, name) is not None] + +for name in eventnames: + globals()[name] = getattr(_c.cConfig, name) + +# ____________________________________________________________ +# poll() +# +class PollError(Exception): + def __init__(self, errno): + self.errno = errno + def get_msg(self): + return os.strerror(self.errno) + +def poll(fddict, timeout=-1): + """'fddict' maps file descriptors to interesting events. + 'timeout' is an integer in milliseconds, and NOT a float + number of seconds, but it's the same in CPython. Use -1 for infinite. + Returns a list [(fd, events)]. + """ + numfd = len(fddict) + pollfds = lltype.malloc(_c.pollfdarray, numfd, flavor='raw') + try: + i = 0 + for fd, events in fddict.iteritems(): + rffi.setintfield(pollfds[i], 'c_fd', fd) + rffi.setintfield(pollfds[i], 'c_events', events) + i += 1 + assert i == numfd + + ret = _c.poll(pollfds, numfd, timeout) + + if ret < 0: + raise PollError(_c.geterrno()) + + retval = [] + for i in range(numfd): + pollfd = pollfds[i] + if pollfd.c_revents: + retval.append((pollfd.c_fd, pollfd.c_revents)) + finally: + lltype.free(pollfds, flavor='raw') + return retval Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Sat Sep 22 17:55:23 2007 @@ -556,7 +556,8 @@ else: rffi.setintfield(pollfd, 'c_events', _c.POLLIN) timeout = int(self.timeout * 1000.0 + 0.5) - n = _c.poll(pollfd, 1, timeout) + n = _c.poll(rffi.cast(lltype.Ptr(_c.pollfdarray), pollfd), + 1, timeout) finally: lltype.free(pollfd, flavor='raw') if n < 0: Added: pypy/dist/pypy/rlib/test/test_rpoll.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/test/test_rpoll.py Sat Sep 22 17:55:23 2007 @@ -0,0 +1,45 @@ +import thread +from pypy.rlib.rsocket import * +from pypy.rlib.rpoll import * + +def test_simple(): + serv = RSocket(AF_INET, SOCK_STREAM) + serv.bind(INETAddress('127.0.0.1', INADDR_ANY)) + serv.listen(1) + servaddr = serv.getsockname() + + events = poll({serv.fileno(): POLLIN}, timeout=100) + assert len(events) == 0 + + cli = RSocket(AF_INET, SOCK_STREAM) + cli.setblocking(False) + err = cli.connect_ex(servaddr) + assert err != 0 + + events = poll({serv.fileno(): POLLIN}, timeout=500) + assert len(events) == 1 + assert events[0][0] == serv.fileno() + assert events[0][1] & POLLIN + + servconn, cliaddr = serv.accept() + + events = poll({serv.fileno(): POLLIN, + cli.fileno(): POLLOUT}, timeout=500) + assert len(events) == 1 + assert events[0][0] == cli.fileno() + assert events[0][1] & POLLOUT + + err = cli.connect_ex(servaddr) + assert err == 0 + + events = poll({servconn.fileno(): POLLIN, + cli.fileno(): POLLIN}, timeout=100) + assert len(events) == 0 + + events = poll({servconn.fileno(): POLLOUT, + cli.fileno(): POLLOUT}, timeout=100) + assert len(events) >= 1 + + cli.close() + servconn.close() + serv.close() From arigo at codespeak.net Sat Sep 22 18:12:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 18:12:35 +0200 (CEST) Subject: [pypy-svn] r46829 - pypy/dist/pypy/rlib/test Message-ID: <20070922161235.038A78100@code0.codespeak.net> Author: arigo Date: Sat Sep 22 18:12:34 2007 New Revision: 46829 Modified: pypy/dist/pypy/rlib/test/test_rpoll.py Log: The test needs this on Windows. Modified: pypy/dist/pypy/rlib/test/test_rpoll.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rpoll.py (original) +++ pypy/dist/pypy/rlib/test/test_rpoll.py Sat Sep 22 18:12:34 2007 @@ -2,6 +2,9 @@ from pypy.rlib.rsocket import * from pypy.rlib.rpoll import * +def setup_module(mod): + rsocket_startup() + def test_simple(): serv = RSocket(AF_INET, SOCK_STREAM) serv.bind(INETAddress('127.0.0.1', INADDR_ANY)) From arigo at codespeak.net Sat Sep 22 18:15:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 18:15:03 +0200 (CEST) Subject: [pypy-svn] r46830 - in pypy/dist/pypy/rlib: . parsing rcairo rcairo/test rsdl rsdl/test test Message-ID: <20070922161503.61D83810E@code0.codespeak.net> Author: arigo Date: Sat Sep 22 18:15:03 2007 New Revision: 46830 Modified: pypy/dist/pypy/rlib/parsing/codebuilder.py (props changed) pypy/dist/pypy/rlib/rcairo/__init__.py (props changed) pypy/dist/pypy/rlib/rcairo/_cairo.py (props changed) pypy/dist/pypy/rlib/rcairo/gen_ll.py (props changed) pypy/dist/pypy/rlib/rcairo/ll_cairo.py (props changed) pypy/dist/pypy/rlib/rcairo/test/ (props changed) pypy/dist/pypy/rlib/rcairo/test/test_ll_cairo.py (props changed) pypy/dist/pypy/rlib/rpoll.py (props changed) pypy/dist/pypy/rlib/rsdl/SDL.py (props changed) pypy/dist/pypy/rlib/rsdl/_SDL.py (props changed) pypy/dist/pypy/rlib/rsdl/__init__.py (props changed) pypy/dist/pypy/rlib/rsdl/ll_SDL.py (props changed) pypy/dist/pypy/rlib/rsdl/test/ (props changed) pypy/dist/pypy/rlib/rsdl/test/test_SDL.py (props changed) pypy/dist/pypy/rlib/rtermios.py (props changed) pypy/dist/pypy/rlib/test/test_rpoll.py (props changed) Log: fixeol From arigo at codespeak.net Sat Sep 22 18:40:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 18:40:19 +0200 (CEST) Subject: [pypy-svn] r46831 - in pypy/dist/pypy: rlib rpython/lltypesystem Message-ID: <20070922164019.7B723810E@code0.codespeak.net> Author: arigo Date: Sat Sep 22 18:40:18 2007 New Revision: 46831 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rpoll.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: poll() for Windows. All this code comes from module/select/interp_select, ported to rffi. The test doesn't pass, but I don't feel like debugging it. It fails in a call to accept() but only if poll() was called previously on that socket... no clue. Needs Windows expertise. Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Sat Sep 22 18:40:18 2007 @@ -490,28 +490,28 @@ [rffi.INT, lltype.Ptr(fd_set), lltype.Ptr(fd_set), lltype.Ptr(fd_set), lltype.Ptr(timeval)], rffi.INT) - - ##WSACreateEvent = socketdll.WSACreateEvent - ##WSACreateEvent.argtypes = [] - ##WSACreateEvent.restype = WSAEVENT - - ##WSACloseEvent = socketdll.WSACloseEvent - ##WSACloseEvent.argtypes = [WSAEVENT] - ##WSACloseEvent.restype = c_int - - ##WSAEventSelect = socketdll.WSAEventSelect - ##WSAEventSelect.argtypes = [socketfd_type, WSAEVENT, rffi.LONG] - ##WSAEventSelect.restype = c_int - - ##WSAWaitForMultipleEvents = socketdll.WSAWaitForMultipleEvents - ##WSAWaitForMultipleEvents.argtypes = [rffi.LONG, POINTER(WSAEVENT), - ## c_int, rffi.LONG, c_int] - ##WSAWaitForMultipleEvents.restype = c_long - - ##WSAEnumNetworkEvents = socketdll.WSAEnumNetworkEvents - ##WSAEnumNetworkEvents.argtypes = [socketfd_type, WSAEVENT, - ## POINTER(WSANETWORKEVENTS)] - ##WSAEnumNetworkEvents.restype = c_int + # + # The following is for pypy.rlib.rpoll + # + WSAEVENT_ARRAY = rffi.CArray(WSAEVENT) + + WSACreateEvent = external('WSACreateEvent', [], WSAEVENT) + + WSACloseEvent = external('WSACloseEvent', [WSAEVENT], rffi.INT) + + WSAEventSelect = external('WSAEventSelect', + [socketfd_type, WSAEVENT, rffi.LONG], + rffi.INT) + + WSAWaitForMultipleEvents = external('WSAWaitForMultipleEvents', + [rffi.LONG, lltype.Ptr(WSAEVENT_ARRAY), + rffi.INT, rffi.LONG, rffi.INT], + rffi.ULONG) + + WSAEnumNetworkEvents = external('WSAEnumNetworkEvents', + [socketfd_type, WSAEVENT, + lltype.Ptr(WSANETWORKEVENTS)], + rffi.INT) if MS_WINDOWS: WSAData = cConfig.WSAData Modified: pypy/dist/pypy/rlib/rpoll.py ============================================================================== --- pypy/dist/pypy/rlib/rpoll.py (original) +++ pypy/dist/pypy/rlib/rpoll.py Sat Sep 22 18:40:18 2007 @@ -21,41 +21,139 @@ for name in eventnames: globals()[name] = getattr(_c.cConfig, name) -# ____________________________________________________________ -# poll() -# class PollError(Exception): def __init__(self, errno): self.errno = errno def get_msg(self): - return os.strerror(self.errno) + return _c.socket_strerror_str(self.errno) + +# ____________________________________________________________ +# poll() for POSIX systems +# +if hasattr(_c, 'poll'): + + def poll(fddict, timeout=-1): + """'fddict' maps file descriptors to interesting events. + 'timeout' is an integer in milliseconds, and NOT a float + number of seconds, but it's the same in CPython. Use -1 for infinite. + Returns a list [(fd, events)]. + """ + numfd = len(fddict) + pollfds = lltype.malloc(_c.pollfdarray, numfd, flavor='raw') + try: + i = 0 + for fd, events in fddict.iteritems(): + rffi.setintfield(pollfds[i], 'c_fd', fd) + rffi.setintfield(pollfds[i], 'c_events', events) + i += 1 + assert i == numfd + + ret = _c.poll(pollfds, numfd, timeout) + + if ret < 0: + raise PollError(_c.geterrno()) + + retval = [] + for i in range(numfd): + pollfd = pollfds[i] + if pollfd.c_revents: + retval.append((pollfd.c_fd, pollfd.c_revents)) + finally: + lltype.free(pollfds, flavor='raw') + return retval + +# ____________________________________________________________ +# poll() for Win32 +# +if hasattr(_c, 'WSAEventSelect'): + + def poll(fddict, timeout=-1): + """'fddict' maps file descriptors to interesting events. + 'timeout' is an integer in milliseconds, and NOT a float + number of seconds, but it's the same in CPython. Use -1 for infinite. + Returns a list [(fd, events)]. + """ + numfd = len(fddict) + numevents = 0 + socketevents = lltype.malloc(_c.WSAEVENT_ARRAY, numfd, flavor='raw') + try: + eventdict = {} + + for fd, events in fddict.iteritems(): + # select desired events + wsaEvents = 0 + if events & _c.POLLIN: + wsaEvents |= _c.FD_READ | _c.FD_ACCEPT | _c.FD_CLOSE + if events & _c.POLLOUT: + wsaEvents |= _c.FD_WRITE | _c.FD_CONNECT | _c.FD_CLOSE + + # if no events then ignore socket + if wsaEvents == 0: + continue + + # select socket for desired events + event = _c.WSACreateEvent() + _c.WSAEventSelect(fd, event, wsaEvents) + + eventdict[fd] = event + socketevents[numevents] = event + numevents += 1 + + assert numevents <= numfd + + # if no sockets then return immediately + # XXX commented out by arigo - we just want to sleep for + # 'timeout' milliseconds in this case, which is what + # I hope WSAWaitForMultipleEvents will do, no? + #if numevents == 0: + # return [] + + # prepare timeout + if timeout < 0: + timeout = _c.INFINITE + + ret = _c.WSAWaitForMultipleEvents(numevents, socketevents, + False, timeout, False) + + if ret == _c.WSA_WAIT_TIMEOUT: + return [] + + if ret == _c.WSA_WAIT_FAILED: + raise PollError(_c.geterrno()) + + retval = [] + info = rffi.make(_c.WSANETWORKEVENTS) + for fd, event in eventdict.iteritems(): + if _c.WSAEnumNetworkEvents(fd, event, info) < 0: + continue + revents = 0 + if info.c_lNetworkEvents & _c.FD_READ: + revents |= _c.POLLIN + if info.c_lNetworkEvents & _c.FD_ACCEPT: + revents |= _c.POLLIN + if info.c_lNetworkEvents & _c.FD_WRITE: + revents |= _c.POLLOUT + if info.c_lNetworkEvents & _c.FD_CONNECT: + if info.c_iErrorCode[_c.FD_CONNECT_BIT]: + revents |= _c.POLLERR + else: + revents |= _c.POLLOUT + if info.c_lNetworkEvents & _c.FD_CLOSE: + if info.c_iErrorCode[_c.FD_CLOSE_BIT]: + revents |= _c.POLLERR + else: + if fddict[fd] & _c.POLLIN: + revents |= _c.POLLIN + if fddict[fd] & _c.POLLOUT: + revents |= _c.POLLOUT + if revents: + retval.append((fd, revents)) + + lltype.free(info, flavor='raw') + + finally: + for i in range(numevents): + _c.WSACloseEvent(socketevents[i]) + lltype.free(socketevents, flavor='raw') -def poll(fddict, timeout=-1): - """'fddict' maps file descriptors to interesting events. - 'timeout' is an integer in milliseconds, and NOT a float - number of seconds, but it's the same in CPython. Use -1 for infinite. - Returns a list [(fd, events)]. - """ - numfd = len(fddict) - pollfds = lltype.malloc(_c.pollfdarray, numfd, flavor='raw') - try: - i = 0 - for fd, events in fddict.iteritems(): - rffi.setintfield(pollfds[i], 'c_fd', fd) - rffi.setintfield(pollfds[i], 'c_events', events) - i += 1 - assert i == numfd - - ret = _c.poll(pollfds, numfd, timeout) - - if ret < 0: - raise PollError(_c.geterrno()) - - retval = [] - for i in range(numfd): - pollfd = pollfds[i] - if pollfd.c_revents: - retval.append((pollfd.c_fd, pollfd.c_revents)) - finally: - lltype.free(pollfds, flavor='raw') - return retval + return retval Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sat Sep 22 18:40:18 2007 @@ -82,8 +82,10 @@ if SOURCE != TARGET: if TARGET is lltype.Float: arg = float(arg) - elif (isinstance(SOURCE, lltype.Number) and - isinstance(TARGET, lltype.Number)): + elif ((isinstance(SOURCE, lltype.Number) + or SOURCE is lltype.Bool) + and (isinstance(TARGET, lltype.Number) + or TARGET is lltype.Bool)): arg = cast(TARGET, arg) real_args = real_args + (arg,) to_free = to_free + (freeme,) From arigo at codespeak.net Sat Sep 22 18:54:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 18:54:32 +0200 (CEST) Subject: [pypy-svn] r46832 - pypy/dist/pypy/module/select Message-ID: <20070922165432.001C580E8@code0.codespeak.net> Author: arigo Date: Sat Sep 22 18:54:32 2007 New Revision: 46832 Removed: pypy/dist/pypy/module/select/_rsocket_ctypes.py Modified: pypy/dist/pypy/module/select/__init__.py pypy/dist/pypy/module/select/interp_select.py Log: Kill _rsocket_ctypes: select works with pypy.rlib.rpoll now. Modified: pypy/dist/pypy/module/select/__init__.py ============================================================================== --- pypy/dist/pypy/module/select/__init__.py (original) +++ pypy/dist/pypy/module/select/__init__.py Sat Sep 22 18:54:32 2007 @@ -13,15 +13,10 @@ } def buildloaders(cls): - constantnames = ''' - POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL - POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG'''.split() - - from _rsocket_ctypes import constants - for name in constantnames: - if name in constants: - value = constants[name] - Module.interpleveldefs[name] = "space.wrap(%r)" % value + from pypy.rlib import rpoll + for name in rpoll.eventnames: + value = getattr(rpoll, name) + Module.interpleveldefs[name] = "space.wrap(%r)" % value super(Module, cls).buildloaders() buildloaders = classmethod(buildloaders) Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Sat Sep 22 18:54:32 2007 @@ -1,12 +1,10 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import W_Root, ObjSpace, interp2app -import _rsocket_ctypes as _c -from ctypes import POINTER, byref -from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError +from pypy.rlib import rpoll -defaultevents = _c.POLLIN | _c.POLLOUT | _c.POLLPRI +defaultevents = rpoll.POLLIN | rpoll.POLLOUT | rpoll.POLLPRI def poll(space): """Returns a polling object, which supports registering and @@ -51,138 +49,27 @@ space.wrap(fd)) unregister.unwrap_spec = ['self', ObjSpace, W_Root] - if hasattr(_c, 'poll'): - def poll(self, space, w_timeout=None): - if space.is_w(w_timeout, space.w_None): - timeout = -1 - else: - timeout = space.int_w(w_timeout) - - numfd = len(self.fddict) - buf = _c.create_string_buffer(_c.sizeof(_c.pollfd) * numfd) - pollfds = _c.cast(buf, POINTER(_c.pollfd)) - i = 0 - for fd, events in self.fddict.iteritems(): - pollfds[i].fd = fd - pollfds[i].events = events - i += 1 - - # XXX Temporary hack for releasing the GIL - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - ret = _c.poll(pollfds, numfd, timeout) - if GIL is not None: GIL.acquire(True) - - if ret < 0: - errno = geterrno() - w_module = space.getbuiltinmodule('select') - w_errortype = space.getattr(w_module, space.wrap('error')) - message = _c.strerror(errno) - raise OperationError(w_errortype, - space.newtuple([space.wrap(errno), - space.wrap(message)])) - - retval_w = [] - for i in range(numfd): - pollfd = pollfds[i] - if pollfd.revents: - retval_w.append(space.newtuple([space.wrap(pollfd.fd), - space.wrap(pollfd.revents)])) - return space.newlist(retval_w) - - elif hasattr(_c, 'WSAEventSelect'): - # win32 implementation - def poll(self, space, w_timeout=None): - numfd = len(self.fddict) - - socketevents = _c.ARRAY(_c.WSAEVENT, numfd)() - - numevents = 0 - eventdict = {} - - for fd, events in self.fddict.iteritems(): - # select desired events - wsaEvents = 0 - if events & _c.POLLIN: - wsaEvents |= _c.FD_READ | _c.FD_ACCEPT | _c.FD_CLOSE - if events & _c.POLLOUT: - wsaEvents |= _c.FD_WRITE | _c.FD_CONNECT | _c.FD_CLOSE - - # if no events then ignore socket - if wsaEvents == 0: - continue - - # select socket for desired events - event = _c.WSACreateEvent() - _c.WSAEventSelect(fd, event, wsaEvents) - - eventdict[fd] = event - socketevents[numevents] = event - numevents += 1 - - # if no sockets then return immediately - if numevents == 0: - return space.newlist([]) - - # prepare timeout - if space.is_w(w_timeout, space.w_None): - timeout = -1 - else: - timeout = space.int_w(w_timeout) - if timeout < 0: - timeout = _c.INFINITE - - # XXX Temporary hack for releasing the GIL - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - ret = _c.WSAWaitForMultipleEvents(numevents, socketevents, - False, timeout, False) - if GIL is not None: GIL.acquire(True) - - if ret == _c.WSA_WAIT_TIMEOUT: - return space.newlist([]) - - if ret < 0: # WSA_WAIT_FAILED is unsigned... - errno = _c.geterrno() - w_module = space.getbuiltinmodule('select') - w_errortype = space.getattr(w_module, space.wrap('error')) - message = _c.socket_strerror(errno) - raise OperationError(w_errortype, - space.newtuple([space.wrap(errno), - space.wrap(message)])) - - retval_w = [] - info = _c.WSANETWORKEVENTS() - for fd, event in eventdict.iteritems(): - if _c.WSAEnumNetworkEvents(fd, event, byref(info)) < 0: - continue - revents = 0 - if info.lNetworkEvents & _c.FD_READ: - revents |= _c.POLLIN - if info.lNetworkEvents & _c.FD_ACCEPT: - revents |= _c.POLLIN - if info.lNetworkEvents & _c.FD_WRITE: - revents |= _c.POLLOUT - if info.lNetworkEvents & _c.FD_CONNECT: - if info.iErrorCode[_c.FD_CONNECT_BIT]: - revents |= _c.POLLERR - else: - revents |= _c.POLLOUT - if info.lNetworkEvents & _c.FD_CLOSE: - if info.iErrorCode[_c.FD_CLOSE_BIT]: - revents |= _c.POLLERR - else: - if self.fddict[fd] & _c.POLLIN: - revents |= _c.POLLIN - if self.fddict[fd] & _c.POLLOUT: - revents |= _c.POLLOUT - if revents: - retval_w.append(space.newtuple([space.wrap(fd), - space.wrap(revents)])) - - _c.WSACloseEvent(event) - - return space.newlist(retval_w) + def poll(self, space, w_timeout=None): + if space.is_w(w_timeout, space.w_None): + timeout = -1 + else: + timeout = space.int_w(w_timeout) + + try: + retval = rpoll.poll(self.fddict, timeout) + except rpoll.PollError, e: + w_module = space.getbuiltinmodule('select') + w_errortype = space.getattr(w_module, space.wrap('error')) + message = e.get_msg() + raise OperationError(w_errortype, + space.newtuple([space.wrap(errno), + space.wrap(message)])) + + retval_w = [] + for fd, revents in retval: + retval_w.append(space.newtuple([space.wrap(fd), + space.wrap(revents)])) + return space.newlist(retval_w) poll.unwrap_spec = ['self', ObjSpace, W_Root] pollmethods = {} From pypy-svn at codespeak.net Sat Sep 22 19:18:12 2007 From: pypy-svn at codespeak.net (Viagra.com Inc) Date: Sat, 22 Sep 2007 19:18:12 +0200 (CEST) Subject: [pypy-svn] September 70% OFF Message-ID: <20070922082017.39779.qmail@p5B13FC15.dip.t-dialin.net> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Sat Sep 22 21:34:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Sep 2007 21:34:44 +0200 (CEST) Subject: [pypy-svn] r46834 - pypy/dist/pypy/module/select Message-ID: <20070922193444.CD33E80F8@code0.codespeak.net> Author: arigo Date: Sat Sep 22 21:34:43 2007 New Revision: 46834 Modified: pypy/dist/pypy/module/select/interp_select.py Log: Typo. Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Sat Sep 22 21:34:43 2007 @@ -62,7 +62,7 @@ w_errortype = space.getattr(w_module, space.wrap('error')) message = e.get_msg() raise OperationError(w_errortype, - space.newtuple([space.wrap(errno), + space.newtuple([space.wrap(e.errno), space.wrap(message)])) retval_w = [] From tismer at codespeak.net Sun Sep 23 01:45:46 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 23 Sep 2007 01:45:46 +0200 (CEST) Subject: [pypy-svn] r46836 - in pypy/dist/pypy: lib/test2 module/_stackless Message-ID: <20070922234546.3C2C880FF@code0.codespeak.net> Author: tismer Date: Sun Sep 23 01:45:45 2007 New Revision: 46836 Modified: pypy/dist/pypy/lib/test2/pickledtasklet.py pypy/dist/pypy/module/_stackless/coroutine.py Log: narrowind down the problem. still not clear about it. Modified: pypy/dist/pypy/lib/test2/pickledtasklet.py ============================================================================== --- pypy/dist/pypy/lib/test2/pickledtasklet.py (original) +++ pypy/dist/pypy/lib/test2/pickledtasklet.py Sun Sep 23 01:45:45 2007 @@ -22,6 +22,8 @@ t.insert() else: t = stackless.tasklet(demo)(14) + print 'before run' stackless.run() + print 'after run' # remark: think of fixing cells etc. on the sprint Modified: pypy/dist/pypy/module/_stackless/coroutine.py ============================================================================== --- pypy/dist/pypy/module/_stackless/coroutine.py (original) +++ pypy/dist/pypy/module/_stackless/coroutine.py Sun Sep 23 01:45:45 2007 @@ -129,6 +129,10 @@ return space.wrap(AppCoroutine._get_state(space).current) w_getcurrent = staticmethod(w_getcurrent) + def w_getmain(space): + return space.wrap(AppCoroutine._get_state(space).main) + w_getmain = staticmethod(w_getmain) + # pickling interface def descr__reduce__(self, space): # this is trying to be simplistic at the moment. @@ -175,7 +179,9 @@ except UnpackValueError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) self.flags = space.int_w(w_flags) - self.parent = space.interp_w(AppCoroutine, w_parent, can_be_None=True) + if space.is_w(w_parent, space.w_None): + w_parent = self.w_getmain(space) + self.parent = space.interp_w(AppCoroutine, w_parent) ec = self.space.getexecutioncontext() self.subctx.setstate(self.space, w_state) self.reconstruct_framechain() @@ -290,6 +296,7 @@ def post_install(module): makeStaticMethod(module, 'coroutine', 'getcurrent') + makeStaticMethod(module, 'coroutine', 'getmain') space = module.space AppCoroutine._get_state(space).post_install() @@ -311,6 +318,7 @@ # but lib/stackless.py uses it _framestack = GetSetProperty(w_descr__framestack), getcurrent = interp2app(AppCoroutine.w_getcurrent), + getmain = interp2app(AppCoroutine.w_getmain), __reduce__ = interp2app(AppCoroutine.descr__reduce__, unwrap_spec=['self', ObjSpace]), __setstate__ = interp2app(AppCoroutine.descr__setstate__, From arigo at codespeak.net Sun Sep 23 17:53:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 23 Sep 2007 17:53:17 +0200 (CEST) Subject: [pypy-svn] r46841 - pypy/dist/pypy/rlib Message-ID: <20070923155317.DDD3D80EF@code0.codespeak.net> Author: arigo Date: Sun Sep 23 17:53:17 2007 New Revision: 46841 Modified: pypy/dist/pypy/rlib/rpoll.py Log: Translation fixes. Modified: pypy/dist/pypy/rlib/rpoll.py ============================================================================== --- pypy/dist/pypy/rlib/rpoll.py (original) +++ pypy/dist/pypy/rlib/rpoll.py Sun Sep 23 17:53:17 2007 @@ -56,8 +56,10 @@ retval = [] for i in range(numfd): pollfd = pollfds[i] - if pollfd.c_revents: - retval.append((pollfd.c_fd, pollfd.c_revents)) + fd = rffi.cast(lltype.Signed, pollfd.c_fd) + revents = rffi.cast(lltype.Signed, pollfd.c_revents) + if revents: + retval.append((fd, revents)) finally: lltype.free(pollfds, flavor='raw') return retval From arigo at codespeak.net Sun Sep 23 17:55:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 23 Sep 2007 17:55:44 +0200 (CEST) Subject: [pypy-svn] r46842 - in pypy/dist/pypy/rpython: . ootypesystem test Message-ID: <20070923155544.7AA0780EF@code0.codespeak.net> Author: arigo Date: Sun Sep 23 17:55:44 2007 New Revision: 46842 Modified: pypy/dist/pypy/rpython/ootypesystem/rdict.py pypy/dist/pypy/rpython/rdict.py pypy/dist/pypy/rpython/test/test_rdict.py Log: Dict iterators must tell that they could raise RuntimeError even in the lltypesystem, even though they don't really do so in the current implementation. This is needed because the annotator thinks that they can (see test). Modified: pypy/dist/pypy/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rdict.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rdict.py Sun Sep 23 17:55:44 2007 @@ -360,10 +360,6 @@ ITER = ootype.DictItemsIterator(KEYTYPE, VALUETYPE) return ootype.Record({"iterator": ITER}) - def _next_implicit_exceptions(self, hop): - hop.has_implicit_exception(StopIteration) - hop.has_implicit_exception(RuntimeError) - def ll_dictiter(ITER, d): iter = ootype.new(ITER) iter.iterator = d.ll_get_items_iterator() Modified: pypy/dist/pypy/rpython/rdict.py ============================================================================== --- pypy/dist/pypy/rpython/rdict.py (original) +++ pypy/dist/pypy/rpython/rdict.py Sun Sep 23 17:55:44 2007 @@ -89,9 +89,6 @@ citerptr = hop.inputconst(lltype.Void, self.lowleveltype) return hop.gendirectcall(self.ll_dictiter, citerptr, v_dict) - def _next_implicit_exceptions(self, hop): - hop.has_implicit_exception(StopIteration) - def rtype_next(self, hop): variant = self.variant v_iter, = hop.inputargs(self) @@ -100,7 +97,9 @@ c1 = hop.inputconst(lltype.Void, None) else: c1 = hop.inputconst(lltype.Void, hop.r_result.lowleveltype) - self._next_implicit_exceptions(hop) # record that we know about it + # record that we know about these two possible exceptions + hop.has_implicit_exception(StopIteration) + hop.has_implicit_exception(RuntimeError) hop.exception_is_here() v = hop.gendirectcall(self.ll_dictnext, v_iter, v_func, c1) if variant == 'keys': Modified: pypy/dist/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rdict.py (original) +++ pypy/dist/pypy/rpython/test/test_rdict.py Sun Sep 23 17:55:44 2007 @@ -686,6 +686,21 @@ assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2) assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3) + def test_resize_during_iteration(self): + def func(): + d = {5: 1, 6: 2, 7: 3} + try: + for key, value in d.iteritems(): + d[key^16] = value*2 + except RuntimeError: + pass + total = 0 + for key in d: + total += key + return total + res = self.interpret(func, []) + assert 5 + 6 + 7 <= res <= 5 + 6 + 7 + (5^16) + (6^16) + (7^16) + # ____________________________________________________________ From tismer at codespeak.net Sun Sep 23 23:25:38 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 23 Sep 2007 23:25:38 +0200 (CEST) Subject: [pypy-svn] r46843 - pypy/dist/pypy/module/_stackless Message-ID: <20070923212538.5752680E0@code0.codespeak.net> Author: tismer Date: Sun Sep 23 23:25:37 2007 New Revision: 46843 Modified: pypy/dist/pypy/module/_stackless/coroutine.py Log: this bug has been bugging me since Vilnius, Hey, now I can really clean-up things :-) Modified: pypy/dist/pypy/module/_stackless/coroutine.py ============================================================================== --- pypy/dist/pypy/module/_stackless/coroutine.py (original) +++ pypy/dist/pypy/module/_stackless/coroutine.py Sun Sep 23 23:25:37 2007 @@ -234,16 +234,15 @@ instr+3, ec) instr += 1 oparg = ord(code[instr]) | ord(code[instr + 1]) << 8 - if (oparg >> 8) & 0xff == 0: + nargs = oparg & 0xff + if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + elif opcode == map['CALL_FUNCTION'] and (oparg >> 8) & 0xff == 0: # Only positional arguments - nargs = oparg & 0xff # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) - if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: - chain = resume_state_create(chain, 'CALL_METHOD', frame, - nargs) - else: - chain = resume_state_create(chain, 'CALL_FUNCTION', frame, - nargs) + chain = resume_state_create(chain, 'CALL_FUNCTION', frame, + nargs) else: # case2: ("call_function", f, returns=w_result) chain = resume_state_create(chain, 'call_function', frame) From arigo at codespeak.net Mon Sep 24 09:48:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 Sep 2007 09:48:01 +0200 (CEST) Subject: [pypy-svn] r46845 - pypy/dist/pypy/translator Message-ID: <20070924074801.3EAC880FC@code0.codespeak.net> Author: arigo Date: Mon Sep 24 09:48:00 2007 New Revision: 46845 Modified: pypy/dist/pypy/translator/exceptiontransform.py Log: Remove an unneeded dependency on ctypes. Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Mon Sep 24 09:48:00 2007 @@ -27,7 +27,7 @@ lltype.Void: None} for TYPE in rffi.NUMBER_TYPES: - PrimitiveErrorValue[TYPE] = rffi.cast(TYPE, -1) + PrimitiveErrorValue[TYPE] = lltype.cast_primitive(TYPE, -1) del TYPE def error_value(T): From arigo at codespeak.net Mon Sep 24 10:41:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 Sep 2007 10:41:42 +0200 (CEST) Subject: [pypy-svn] r46846 - in pypy/dist/pypy: objspace/cpy/test translator/c/src Message-ID: <20070924084142.AC72380FC@code0.codespeak.net> Author: arigo Date: Mon Sep 24 10:41:41 2007 New Revision: 46846 Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py pypy/dist/pypy/translator/c/src/mem.h Log: Try to use the official way to build PyObjects, instead of the broken hack that forgot about CPython's GC. Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_typedef.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_typedef.py Mon Sep 24 10:41:41 2007 @@ -48,7 +48,7 @@ fn = compile(make_mytype, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' @@ -70,9 +70,9 @@ fn = compile(make_mytype, [int], annotatorpolicy = CPyAnnotatorPolicy(space)) - res2 = fn(1, expected_extra_mallocs=1) + res2 = fn(1) assert type(res2).__name__ == 'MyType2' - res = fn(0, expected_extra_mallocs=2) + res = fn(0) assert type(res).__name__ == 'MyType' @@ -99,19 +99,19 @@ fn = compile(fn, [object], annotatorpolicy = CPyAnnotatorPolicy(space)) - res, abc = fn(None, expected_extra_mallocs=1) + res, abc = fn(None) assert abc == 2 assert type(res).__name__ == 'MyType' - res2, abc = fn(res, expected_extra_mallocs=1) + res2, abc = fn(res) assert abc == 4 assert res2 is res - res2, abc = fn(res, expected_extra_mallocs=1) + res2, abc = fn(res) assert abc == 8 assert res2 is res - res2, abc = fn(res, expected_extra_mallocs=1) + res2, abc = fn(res) assert abc == 16 assert res2 is res @@ -126,7 +126,7 @@ fn = compile(make_mytype, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' assert res.hello == 7 assert type(res).hello == 7 @@ -143,7 +143,7 @@ fn = compile(make_mytype, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' assert res.multiply(3) == 369 @@ -158,7 +158,7 @@ fn = compile(make_mytype, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' assert res * 3 == 369 @@ -187,12 +187,12 @@ fn = compile(fn, [object], annotatorpolicy = CPyAnnotatorPolicy(space)) - res, x = fn(None, expected_extra_mallocs=1) + res, x = fn(None) assert type(res).__name__ == 'MyType' assert x == 2 assert res.x == 2 - res2, x = fn(res, expected_extra_mallocs=1) + res2, x = fn(res) assert res2 is res assert x == 4 assert res.x == 4 @@ -223,14 +223,14 @@ fn = compile(fn, [object], annotatorpolicy = CPyAnnotatorPolicy(space)) - res, x = fn(None, expected_extra_mallocs=1) + res, x = fn(None) assert type(res).__name__ == 'MyType' assert x == 2 assert res.x == 2 res.x += 100 assert res.x == 102 - res2, x = fn(res, expected_extra_mallocs=1) + res2, x = fn(res) assert res2 is res assert x == 204 assert res.x == 204 @@ -288,7 +288,7 @@ fn = compile(build, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' assert res.x == 42 @@ -315,7 +315,7 @@ fn = compile(build, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' assert res.x == 42 @@ -338,7 +338,7 @@ fn = compile(build, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' assert res.x == 42 @@ -372,7 +372,7 @@ fn = compile(build, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=1) + res = fn() assert type(res).__name__ == 'MyType' def test_prebuilt_instance(): @@ -394,7 +394,7 @@ fn = compile(build, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=0) + res = fn() assert type(res).__name__ == 'MyType' def test_prebuilt_instance_inside_pyobj(): @@ -415,7 +415,7 @@ fn = compile(build, [], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(expected_extra_mallocs=0) + res = fn() assert type(res) is dict assert res.keys() == ['hello'] assert type(res['hello']).__name__ == 'MyType' @@ -432,5 +432,5 @@ fn = compile(entry_point, [int], annotatorpolicy = CPyAnnotatorPolicy(space)) - res = fn(42, expected_extra_mallocs=1) + res = fn(42) assert res.hello == 7 Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Mon Sep 24 10:41:41 2007 @@ -113,15 +113,12 @@ /************************************************************/ /* rcpy support */ -#define OP_CPY_MALLOC(cpytype, r, restype) { \ - /* XXX add tp_itemsize later */ \ - OP_RAW_MALLOC(((PyTypeObject *)cpytype)->tp_basicsize, r, restype); \ - if (r) { \ - OP_RAW_MEMCLEAR(r, ((PyTypeObject *)cpytype)->tp_basicsize, /* */); \ - PyObject_Init((PyObject *)r, (PyTypeObject *)cpytype); \ - } \ +#define OP_CPY_MALLOC(cpytype, r, restype) { \ + /* XXX add tp_itemsize later */ \ + r = ((PyTypeObject *)cpytype)->tp_alloc((PyTypeObject *)cpytype, 0); \ + if (!r) RPyConvertExceptionFromCPython(); \ } -#define OP_CPY_FREE(x) OP_RAW_FREE(x, /*nothing*/) +#define OP_CPY_FREE(x) XXX "this shouldn't be used any more" /************************************************************/ /* weakref support */ From simonb at codespeak.net Tue Sep 25 03:23:28 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Tue, 25 Sep 2007 03:23:28 +0200 (CEST) Subject: [pypy-svn] r46855 - in pypy/dist/pypy/rpython/numpy: . test Message-ID: <20070925012328.1330E810E@code0.codespeak.net> Author: simonb Date: Tue Sep 25 03:23:27 2007 New Revision: 46855 Modified: pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/numpy/test/test_array.py Log: add some tests; skip for now Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Tue Sep 25 03:23:27 2007 @@ -20,7 +20,6 @@ GcArray, GcStruct, Number, Primitive, Signed, Ptr, Unsigned, Char, Void, FixedSizeArray, Bool,\ GcForwardReference, malloc, direct_arrayitems, direct_ptradd, nullptr, typeMethod,\ cast_primitive -from pypy.rpython.lltypesystem.rffi import cast from pypy.rpython.lltypesystem.rtuple import TupleRepr def gen_build_from_shape(ndim, zero=False): @@ -647,6 +646,7 @@ r_item, v_item = convert_scalar_to_array(r_item, v_item, hop.llops) source_ndim = 1 elif isinstance(r_item, AbstractBaseListRepr): + # Note this does a copy: r_item, v_item = convert_list_to_array(r_item, v_item, hop.llops) source_ndim = 1 else: Modified: pypy/dist/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/dist/pypy/rpython/numpy/test/test_array.py Tue Sep 25 03:23:27 2007 @@ -829,6 +829,37 @@ for i in range(len(data)): assert res.dataptr[i] == data[i]//2 + def test_specialize_array_setitem_alias(self): + py.test.skip('not implemented') + def f(): + a = numpy.array([1,2,3,4]) + a[1:] = a[:3] + return a + + res = interpret(f, []) + data = [1,1,2,3] + for i in range(len(data)): + assert res.dataptr[i] == data[i] + + def test_specialize_array_inplace_add_alias(self): + py.test.skip('not implemented') + def f(): + a = numpy.array([1,2,3,4]) + a[1:] += a[:3] + return a + + res = interpret(f, []) + data = [1,3,5,7] + for i in range(len(data)): + assert res.dataptr[i] == data[i] + +""" +TODO (maybe): +* indexing with ellipses +* indexing with newaxis +* unary plus operator (does a copy) +* advanced selection: indexing with bool/int arrays (or lists) +""" class Test_compile: def setup_class(self): From arigo at codespeak.net Tue Sep 25 12:26:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 12:26:41 +0200 (CEST) Subject: [pypy-svn] r46860 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070925102641.A1D788111@code0.codespeak.net> Author: arigo Date: Tue Sep 25 12:26:40 2007 New Revision: 46860 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Bug, test and fix for the lifetime of raw-malloced data structures passed to C code. The ALLOCATED dictionary could also be used to detect leaks in more places now. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Tue Sep 25 12:26:40 2007 @@ -252,6 +252,8 @@ # ____________________________________________________________ # Ctypes-aware subclasses of the _parentable classes +ALLOCATED = {} # mapping {address: _container} + def get_common_subclass(cls1, cls2, cache={}): """Return a unique subclass with (cls1, cls2) as bases.""" try: @@ -260,7 +262,6 @@ subcls = type('_ctypes_%s' % (cls1.__name__,), (cls1, cls2), {'__slots__': ()}) - assert '__dict__' not in dir(subcls) # use __slots__ everywhere cache[cls1, cls2] = subcls return subcls @@ -279,6 +280,24 @@ """ __slots__ = () + def _ctypes_storage_was_allocated(self): + addr = ctypes.addressof(self._storage) + if addr in ALLOCATED: + raise Exception("internal ll2ctypes error - " + "double conversion from lltype to ctypes?") + ALLOCATED[addr] = self + + def _free(self): + self._check() # no double-frees + # allow the ctypes object to go away now + addr = ctypes.addressof(self._storage) + try: + del ALLOCATED[addr] + except KeyError: + raise Exception("invalid free() - data already freed or " + "not allocated from RPython at all") + self._storage = None + def __eq__(self, other): if not isinstance(other, lltype._parentable): return False @@ -377,6 +396,9 @@ return lltype2ctypes(llres) return ctypes_func_type(callback) + if T.TO._gckind != 'raw': + raise Exception("can only pass 'raw' data structures to C, not %r" + % (T.TO._gckind,)) if container._storage is None: raise RuntimeError("attempting to pass a freed structure to C") if container._storage is True: @@ -386,10 +408,11 @@ elif isinstance(T.TO, lltype.Array): convert_array(container) elif isinstance(T.TO, lltype.OpaqueType): - container._storage = ctypes.create_string_buffer( - T.TO.hints['getsize']()) + cbuf = ctypes.create_string_buffer(T.TO.hints['getsize']()) + add_storage(container, _parentable_mixin, cbuf) else: raise NotImplementedError(T) + container._ctypes_storage_was_allocated() storage = container._storage p = ctypes.pointer(storage) if normalize and hasattr(storage, '_normalized_ctype'): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Tue Sep 25 12:26:40 2007 @@ -5,11 +5,15 @@ from pypy.rpython.lltypesystem.ll2ctypes import lltype2ctypes, ctypes2lltype from pypy.rpython.lltypesystem.ll2ctypes import standard_c_lib from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes +from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.annlowlevel import llhelper class TestLL2Ctypes(object): + def setup_method(self, meth): + ALLOCATED.clear() + def test_primitive(self): assert lltype2ctypes(5) == 5 assert lltype2ctypes('?') == ord('?') @@ -35,6 +39,7 @@ py.test.raises(ValueError, 'cptr.contents') # NULL pointer access res = ctypes2lltype(lltype.Ptr(S), cptr) assert res == p + assert not ALLOCATED # detects memory leaks in the test def test_simple_struct(self): S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Signed)) @@ -50,6 +55,7 @@ s.y = 52 assert sc.contents.y == 52 lltype.free(s, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_struct_ptrs(self): S2 = lltype.Struct('S2', ('y', lltype.Signed)) @@ -72,6 +78,7 @@ lltype.free(s1, flavor='raw') lltype.free(s2a, flavor='raw') lltype.free(s2b, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_simple_array(self): A = lltype.Array(lltype.Signed) @@ -88,6 +95,7 @@ a[3] = 789 assert ac.contents.items[3] == 789 lltype.free(a, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_array_nolength(self): A = lltype.Array(lltype.Signed, hints={'nolength': True}) @@ -104,6 +112,7 @@ assert ac.contents.items[3] == 789 assert ctypes.sizeof(ac.contents) == 10 * ctypes.sizeof(ctypes.c_long) lltype.free(a, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_charp(self): s = rffi.str2charp("hello") @@ -119,6 +128,8 @@ assert s[1] == 'E' s[0] = 'H' assert sc.contents.items[0] == ord('H') + rffi.free_charp(s) + assert not ALLOCATED # detects memory leaks in the test def test_strlen(self): strlen = rffi.llexternal('strlen', [rffi.CCHARP], rffi.SIZE_T, @@ -131,6 +142,7 @@ res = strlen(s) rffi.free_charp(s) assert res == 0 # actually r_size_t(0) + assert not ALLOCATED # detects memory leaks in the test def test_func_not_in_clib(self): foobar = rffi.llexternal('I_really_dont_exist', [], lltype.Signed) @@ -147,6 +159,7 @@ foobar = rffi.llexternal('I_really_dont_exist', [], lltype.Signed, libraries=['I_really_dont_exist_either']) py.test.raises(NotImplementedError, foobar) + assert not ALLOCATED # detects memory leaks in the test def test_cstruct_to_ll(self): S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Signed)) @@ -175,6 +188,7 @@ assert t.x == 125 lltype.free(s, flavor='raw') lltype.free(s2, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_carray_to_ll(self): A = lltype.Array(lltype.Signed, hints={'nolength': True}) @@ -205,6 +219,7 @@ assert b[2] == 660 lltype.free(a, flavor='raw') lltype.free(a2, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_strchr(self): strchr = rffi.llexternal('strchr', [rffi.CCHARP, rffi.INT], @@ -218,6 +233,7 @@ assert res[3] == '\x00' # XXX maybe we should also allow res[-1], res[-2]... rffi.free_charp(s) + assert not ALLOCATED # detects memory leaks in the test def test_frexp(self): A = lltype.FixedSizeArray(rffi.INT, 1) @@ -230,6 +246,7 @@ assert res == 0.625 assert p[0] == 2 lltype.free(p, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_rand(self): rand = rffi.llexternal('rand', [], rffi.INT, @@ -247,6 +264,7 @@ assert res1 == res1b assert res2 == res2b assert res3 == res3b + assert not ALLOCATED # detects memory leaks in the test def test_opaque_obj(self): includes = ['sys/time.h', 'time.h'] @@ -258,6 +276,9 @@ ll_timezonep = lltype.malloc(TIMEZONEP.TO, flavor='raw') res = gettimeofday(ll_timevalp, ll_timezonep) assert res != -1 + lltype.free(ll_timezonep, flavor='raw') + lltype.free(ll_timevalp, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_simple_cast(self): assert rffi.cast(rffi.SIGNEDCHAR, 0x123456) == 0x56 @@ -265,6 +286,7 @@ assert rffi.cast(rffi.CHAR, 0x123456) == '\x56' assert rffi.cast(rffi.CHAR, 0x123481) == '\x81' assert rffi.cast(rffi.UCHAR, 0x123481) == 0x81 + assert not ALLOCATED # detects memory leaks in the test def test_forced_ptr_cast(self): import array @@ -295,6 +317,7 @@ assert e[i] == i*i lltype.free(a, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_funcptr1(self): def dummy(n): @@ -310,6 +333,7 @@ assert lltype.typeOf(lldummy) == lltype.Ptr(FUNCTYPE) res = lldummy(41) assert res == 42 + assert not ALLOCATED # detects memory leaks in the test def test_funcptr2(self): FUNCTYPE = lltype.FuncType([rffi.CCHARP], lltype.Signed) @@ -325,6 +349,7 @@ res = cstrlen2(cp) assert res == 8 rffi.free_charp(p) + assert not ALLOCATED # detects memory leaks in the test def test_qsort(self): CMPFUNC = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], rffi.INT) @@ -360,6 +385,7 @@ for i in range(10): assert a[i] == lst[i] lltype.free(a, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test # def test_signal(self):... @@ -391,6 +417,8 @@ sc = lltype2ctypes(s) checkval(sc.contents.x, 'l') checkval(sc.contents.y, 'l') + lltype.free(s, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_substructures(self): S1 = lltype.Struct('S1', ('x', lltype.Signed)) @@ -439,6 +467,7 @@ t1.x += 1 assert sc.contents.s1a.x == 60 lltype.free(s, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_recursive_struct(self): SX = lltype.ForwardReference() @@ -495,6 +524,7 @@ ctypes.addressof(sc1.contents)) lltype.free(s1, flavor='raw') lltype.free(s2, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_indirect_recursive_struct(self): S2Forward = lltype.ForwardReference() @@ -516,6 +546,7 @@ lltype.free(s1, flavor='raw') lltype.free(a2, flavor='raw') lltype.free(s2, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_arrayofstruct(self): S1 = lltype.Struct('S1', ('x', lltype.Signed)) @@ -541,6 +572,7 @@ assert aitem1.x == 101 assert aitem1 == a1[1] lltype.free(a, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test def test_get_errno(self): if sys.platform.startswith('win'): @@ -562,6 +594,7 @@ err = rffi.get_errno() import errno assert err == errno.EBADF + assert not ALLOCATED # detects memory leaks in the test def test_call_with_struct_argument(self): # XXX is there such a function in the standard C headers? @@ -574,3 +607,21 @@ p = _rsocket_rffi.inet_ntoa(buf) assert rffi.charp2str(p) == '1.2.3.4' lltype.free(buf, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test + + def test_storage_stays_around(self): + data = "hello, world!" * 100 + A = lltype.Array(rffi.CHAR, hints={'nolength': True}) + S = lltype.Struct('S', ('a', lltype.Ptr(A))) + s = lltype.malloc(S, flavor='raw') + lltype2ctypes(s) # force it to escape + s.a = lltype.malloc(A, len(data), flavor='raw') + # the storage for the array should not be freed by lltype even + # though the _ptr object appears to go away here + for i in xrange(len(data)): + s.a[i] = data[i] + for i in xrange(len(data)): + assert s.a[i] == data[i] + lltype.free(s.a, flavor='raw') + lltype.free(s, flavor='raw') + assert not ALLOCATED # detects memory leaks in the test From arigo at codespeak.net Tue Sep 25 14:43:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 14:43:02 +0200 (CEST) Subject: [pypy-svn] r46861 - in pypy/dist/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/c/test Message-ID: <20070925124302.378268141@code0.codespeak.net> Author: arigo Date: Tue Sep 25 14:43:00 2007 New Revision: 46861 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py pypy/dist/pypy/translator/c/test/test_lltyped.py Log: Reuse the direct_ptradd operation to also work directly on "barebone" arrays, i.e. raw-malloced length-less arrays. This is semi-natural because it works on such arrays in the C backend out of the box. This is only implemented via ll2ctypes at the moment - no proper lltype emulation. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Tue Sep 25 14:43:00 2007 @@ -622,6 +622,43 @@ TYPE1 = v_arg.concretetype return gen_cast(hop.llops, RESTYPE, v_arg) +def typecheck_ptradd(T): + # --- ptradd() is only for pointers to non-GC, no-length arrays. + assert isinstance(T, lltype.Ptr) + assert isinstance(T.TO, lltype.Array) + assert T.TO._hints.get('nolength') + +def force_ptradd(ptr, n): + """'ptr' must be a pointer to an array. Equivalent of 'ptr + n' in + C, i.e. gives a pointer to the n'th item of the array. The type of + the result is again a pointer to an array, the same as the type of + 'ptr'. + """ + T = lltype.typeOf(ptr) + typecheck_ptradd(T) + ctypes_item_type = get_ctypes_type(T.TO.OF) + ctypes_arrayptr_type = get_ctypes_type(T) + cptr = lltype2ctypes(ptr) + baseaddr = ctypes.addressof(cptr.contents.items) + addr = baseaddr + n * ctypes.sizeof(ctypes_item_type) + cptr = ctypes.cast(ctypes.c_void_p(addr), ctypes_arrayptr_type) + return ctypes2lltype(T, cptr) + +class ForceCastEntry(ExtRegistryEntry): + _about_ = force_ptradd + + def compute_result_annotation(self, s_ptr, s_n): + assert isinstance(s_n, annmodel.SomeInteger) + assert isinstance(s_ptr, annmodel.SomePtr) + typecheck_ptradd(s_ptr.ll_ptrtype) + return annmodel.lltype_to_annotation(s_ptr.ll_ptrtype) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + v_ptr, v_n = hop.inputargs(hop.args_r[0], lltype.Signed) + return hop.genop('direct_ptradd', [v_ptr, v_n], + resulttype = v_ptr.concretetype) + # ____________________________________________________________ # errno Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Tue Sep 25 14:43:00 2007 @@ -831,12 +831,15 @@ def direct_ptradd(ptr, n): """Shift a pointer forward or backward by n items. The pointer must - have been built by direct_arrayitems(). + have been built by direct_arrayitems(), or it must be directly a + pointer to a raw array with no length (handled by emulation with ctypes). """ if not ptr: raise RuntimeError("direct_ptradd: NULL argument") if not isinstance(ptr._obj, _subarray): - raise TypeError("direct_ptradd: only for direct_arrayitems() ptrs") + # special case: delegate barebone C-like array cases to rffi.ptradd() + from pypy.rpython.lltypesystem import rffi + return rffi.ptradd(ptr, n) parent, base = parentlink(ptr._obj) return _subarray._makeptr(parent, base + n, ptr._solid) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Tue Sep 25 14:43:00 2007 @@ -335,6 +335,8 @@ cast = ll2ctypes.force_cast # a forced, no-checking cast +ptradd = ll2ctypes.force_ptradd # equivalent of "ptr + n" in C. + # the ptr must point to an array. def size_and_sign(tp): size = sizeof(tp) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Tue Sep 25 14:43:00 2007 @@ -442,3 +442,20 @@ finally: os.close(write_fd) os.close(read_fd) + + +ARRAY_OF_CHAR = lltype.Array(CHAR, hints={'nolength': True}) + +def test_ptradd(): + data = "hello, world!" + a = lltype.malloc(ARRAY_OF_CHAR, len(data), flavor='raw') + for i in xrange(len(data)): + a[i] = data[i] + a2 = ptradd(a, 2) + assert lltype.typeOf(a2) == lltype.typeOf(a) == lltype.Ptr(ARRAY_OF_CHAR) + for i in xrange(len(data) - 2): + assert a2[i] == a[i + 2] + lltype.free(a, flavor='raw') + +def test_ptradd_interpret(): + interpret(test_ptradd, []) Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Tue Sep 25 14:43:00 2007 @@ -580,3 +580,21 @@ 0, maxlonglong*2-30, # lshift 0, maxlonglong>>4, # rshift )) + + def test_direct_ptradd_barebone(self): + from pypy.rpython.lltypesystem import rffi + ARRAY_OF_CHAR = Array(Char, hints={'nolength': True}) + + def llf(): + data = "hello, world!" + a = malloc(ARRAY_OF_CHAR, len(data), flavor='raw') + for i in xrange(len(data)): + a[i] = data[i] + a2 = rffi.ptradd(a, 2) + assert typeOf(a2) == typeOf(a) == Ptr(ARRAY_OF_CHAR) + for i in xrange(len(data) - 2): + assert a2[i] == a[i + 2] + free(a, flavor='raw') + + fn = self.getcompiled(llf) + fn() From arigo at codespeak.net Tue Sep 25 14:47:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 14:47:37 +0200 (CEST) Subject: [pypy-svn] r46862 - in pypy/dist/pypy/rlib: . test Message-ID: <20070925124737.BB7038141@code0.codespeak.net> Author: arigo Date: Tue Sep 25 14:47:37 2007 New Revision: 46862 Modified: pypy/dist/pypy/rlib/rsocket.py pypy/dist/pypy/rlib/test/test_rsocket.py Log: Use rffi.ptradd() to write a reasonable implementation of socket.sendall(). Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Tue Sep 25 14:47:37 2007 @@ -843,28 +843,43 @@ lltype.free(buf, flavor='raw') raise self.error_handler() - def send(self, data, flags=0): - """Send a data string to the socket. For the optional flags - argument, see the Unix manual. Return the number of bytes - sent; this may be less than len(data) if the network is busy.""" + def send_raw(self, dataptr, length, flags=0): + """Send data from a CCHARP buffer.""" res = -1 timeout = self._select(False) if timeout == 1: raise SocketTimeout elif timeout == 0: - res = _c.send(self.fd, data, len(data), flags) + res = _c.send(self.fd, dataptr, length, flags) if res < 0: raise self.error_handler() return res + def send(self, data, flags=0): + """Send a data string to the socket. For the optional flags + argument, see the Unix manual. Return the number of bytes + sent; this may be less than len(data) if the network is busy.""" + dataptr = rffi.str2charp(data) + try: + return self.send_raw(dataptr, len(data), flags) + finally: + rffi.free_charp(dataptr) + def sendall(self, data, flags=0): """Send a data string to the socket. For the optional flags argument, see the Unix manual. This calls send() repeatedly until all data is sent. If an error occurs, it's impossible to tell how much data has been sent.""" - while data: - res = self.send(data, flags) - data = data[res:] + dataptr = rffi.str2charp(data) + try: + remaining = len(data) + p = dataptr + while remaining > 0: + res = self.send_raw(p, remaining, flags) + p = rffi.ptradd(p, res) + remaining -= res + finally: + rffi.free_charp(dataptr) def sendto(self, data, flags, address): """Like send(data, flags) but allows specifying the destination Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Tue Sep 25 14:47:37 2007 @@ -91,7 +91,7 @@ if sys.platform == "win32": py.test.skip('No socketpair on Windows') s1, s2 = socketpair() - s1.send('?') + s1.sendall('?') buf = s2.recv(100) assert buf == '?' count = s2.send('x'*99) From arigo at codespeak.net Tue Sep 25 17:21:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 17:21:40 +0200 (CEST) Subject: [pypy-svn] r46863 - in pypy/dist/pypy/rlib: . test Message-ID: <20070925152140.AD15B8150@code0.codespeak.net> Author: arigo Date: Tue Sep 25 17:21:39 2007 New Revision: 46863 Added: pypy/dist/pypy/rlib/rzlib.py - copied, changed from r46859, pypy/branch/zlib-rffi-impl/pypy/module/zlib/interp_zlib.py pypy/dist/pypy/rlib/test/test_rzlib.py - copied, changed from r46859, pypy/branch/zlib-rffi-impl/pypy/module/zlib/test/test_zlib.py Log: Copy the zlib code from exarkun's branch zlib-rffi-impl. The goal is to separate the RPython implementation based on rffi (living in this new pypy.rlib.rzlib) from the PyPy app-level module. From arigo at codespeak.net Tue Sep 25 18:25:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 18:25:42 +0200 (CEST) Subject: [pypy-svn] r46864 - in pypy/dist/pypy/rlib: . test Message-ID: <20070925162542.21F1C816E@code0.codespeak.net> Author: arigo Date: Tue Sep 25 18:25:40 2007 New Revision: 46864 Modified: pypy/dist/pypy/rlib/rzlib.py pypy/dist/pypy/rlib/test/test_rzlib.py Log: Reimplement compression, done in a loop and accumulating all output data. Modified: pypy/dist/pypy/rlib/rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/rzlib.py (original) +++ pypy/dist/pypy/rlib/rzlib.py Tue Sep 25 18:25:40 2007 @@ -4,6 +4,13 @@ includes = ['zlib.h'] libraries = ['z'] +constantnames = ''' + Z_OK Z_STREAM_ERROR Z_BUF_ERROR Z_MEM_ERROR Z_STREAM_END + Z_DEFLATED Z_DEFAULT_STRATEGY Z_DEFAULT_COMPRESSION + Z_NO_FLUSH Z_FINISH + MAX_WBITS MAX_MEM_LEVEL + '''.split() + class SimpleCConfig: """ Definitions for basic types defined by zlib. @@ -17,20 +24,10 @@ Bytef = rffi_platform.SimpleType('Bytef', rffi.UCHAR) voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP) - Z_OK = rffi_platform.ConstantInteger('Z_OK') - Z_STREAM_ERROR = rffi_platform.ConstantInteger('Z_STREAM_ERROR') - ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION') - Z_DEFLATED = rffi_platform.ConstantInteger('Z_DEFLATED') - Z_DEFAULT_STRATEGY = rffi_platform.ConstantInteger('Z_DEFAULT_STRATEGY') - Z_DEFAULT_COMPRESSION = rffi_platform.ConstantInteger( - 'Z_DEFAULT_COMPRESSION') - Z_NO_FLUSH = rffi_platform.ConstantInteger( - 'Z_NO_FLUSH') - - MAX_WBITS = rffi_platform.ConstantInteger('MAX_WBITS') - MAX_MEM_LEVEL = rffi_platform.ConstantInteger('MAX_MEM_LEVEL') +for _name in constantnames: + setattr(SimpleCConfig, _name, rffi_platform.ConstantInteger(_name)) config = rffi_platform.configure(SimpleCConfig) voidpf = config['voidpf'] @@ -39,18 +36,10 @@ Bytef = config['Bytef'] Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True})) -Z_OK = config['Z_OK'] -Z_STREAM_ERROR = config['Z_STREAM_ERROR'] - ZLIB_VERSION = config['ZLIB_VERSION'] -Z_DEFAULT_COMPRESSION = config['Z_DEFAULT_COMPRESSION'] -Z_DEFAULT_STRATEGY = config['Z_DEFAULT_STRATEGY'] -Z_NO_FLUSH = config['Z_DEFAULT_COMPRESSION'] -Z_DEFLATED = config['Z_DEFLATED'] - -MAX_WBITS = config['MAX_WBITS'] -MAX_MEM_LEVEL = config['MAX_MEM_LEVEL'] +for _name in constantnames: + globals()[_name] = config[_name] # The following parameter is copied from zutil.h, version 0.95, # according to CPython's zlibmodule.c @@ -59,6 +48,8 @@ else: DEF_MEM_LEVEL = MAX_MEM_LEVEL +OUTPUT_BUFFER_SIZE = 32*1024 + class ComplexCConfig: """ @@ -153,17 +144,16 @@ rffi.free_charp(bytes) return checksum - -def _make_stream(): - """Helper to build a stream.""" - return lltype.malloc(z_stream, flavor='raw', zero=True) - +# ____________________________________________________________ class RZlibError(Exception): """Exception raised by failing operations in pypy.rlib.rzlib.""" def __init__(self, msg): self.msg = msg + def __str__(self): + return self.msg + def fromstream(stream, err, while_doing): """Return a RZlibError with a message formatted from a zlib error code and stream. @@ -190,7 +180,7 @@ Allocate and return an opaque 'stream' object that can be used to compress data. """ - stream = _make_stream() + stream = lltype.malloc(z_stream, flavor='raw', zero=True) err = _deflateInit2(stream, level, method, wbits, memLevel, strategy) if err == Z_OK: return stream @@ -219,180 +209,66 @@ lltype.free(stream, flavor='raw') -##class _StreamBase(Wrappable): -## """ -## Base for classes which want to have a z_stream allocated when they are -## initialized and de-allocated when they are freed. -## """ -## def __init__(self, space): -## self.space = space -## self.stream = lltype.malloc(z_stream, flavor='raw') -## self.stream.c_zalloc = lltype.nullptr(z_stream.c_zalloc.TO) -## self.stream.c_zfree = lltype.nullptr(z_stream.c_zfree.TO) -## self.stream.c_avail_in = rffi.cast(lltype.Unsigned, 0) -## self.stream.c_next_in = lltype.nullptr(z_stream.c_next_in.TO) - - -## def __del__(self): -## lltype.free(self.stream, flavor='raw') - - - -##def error_from_zlib(space, status): -## if status == Z_STREAM_ERROR: -## return OperationError( -## space.w_ValueError, -## space.wrap("Invalid initialization option")) -## assert False, "unhandled status %s" % (status,) - - -##class Compress(_StreamBase): -## """ -## Wrapper around zlib's z_stream structure which provides convenient -## compression functionality. -## """ -## def __init__(self, space, w_level): -## # XXX CPython actually exposes 4 more undocumented parameters beyond -## # level. -## if space.is_w(w_level, space.w_None): -## level = Z_DEFAULT_COMPRESSION -## else: -## level = space.int_w(w_level) - -## _StreamBase.__init__(self, space) - -## method = Z_DEFLATED -## windowBits = 15 -## memLevel = 8 -## strategy = Z_DEFAULT_STRATEGY - -## result = _deflateInit2( -## self.stream, level, method, windowBits, memLevel, strategy) - -## if result != Z_OK: -## raise error_from_zlib(self.space, result) - - -## def compress(self, data, length=16384): -## """ -## compress(data) -- Return a string containing data compressed. - -## After calling this function, some of the input data may still be stored -## in internal buffers for later processing. - -## Call the flush() method to clear these buffers. -## """ -## self.stream.c_avail_in = rffi.cast(lltype.Unsigned, len(data)) -## self.stream.c_next_in = lltype.malloc( -## Bytefp.TO, len(data), flavor='raw') -## for i in xrange(len(data)): -## self.stream.c_next_in[i] = rffi.cast(Bytef, data[i]) -## self.stream.c_avail_out = rffi.cast(lltype.Unsigned, length) -## self.stream.c_next_out = lltype.malloc( -## Bytefp.TO, length, flavor='raw') -## result = _deflate(self.stream, Z_NO_FLUSH) -## if result != Z_OK: -## raise error_from_zlib(self.space, result) -## return rffi.charp2str(self.stream.c_next_out) -## compress.unwrap_spec = ['self', str, int] - - -## def flush(self, mode=0): # XXX =Z_FINISH -## """ -## flush( [mode] ) -- Return a string containing any remaining compressed -## data. - -## mode can be one of the constants Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH; -## the default value used when mode is not specified is Z_FINISH. - -## If mode == Z_FINISH, the compressor object can no longer be used after -## calling the flush() method. Otherwise, more data can still be -## compressed. -## """ -## return self.space.wrap('') -## flush.unwrap_spec = ['self', int] - - -##def Compress___new__(space, w_subtype, w_level=None): -## """ -## Create a new z_stream and call its initializer. -## """ -## stream = space.allocate_instance(Compress, w_subtype) -## stream = space.interp_w(Compress, stream) -## Compress.__init__(stream, space, w_level) -## return space.wrap(stream) -##Compress___new__.unwrap_spec = [ObjSpace, W_Root, W_Root] - - -##Compress.typedef = TypeDef( -## 'Compress', -## __new__ = interp2app(Compress___new__), -## compress = interp2app(Compress.compress), -## flush = interp2app(Compress.flush)) - - - -##class Decompress(_StreamBase): -## """ -## Wrapper around zlib's z_stream structure which provides convenient -## decompression functionality. -## """ -## def __init__(self, space, wbits): -## """ -## Initialize a new decompression object. - -## wbits is an integer between 8 and MAX_WBITS or -8 and -MAX_WBITS -## (inclusive) giving the number of "window bits" to use for compression -## and decompression. See the documentation for deflateInit2 and -## inflateInit2. -## """ -## _StreamBase.__init__(self, space) -## self.wbits = wbits - - -## def decompress(self, data, max_length=0): # XXX =None -## """ -## decompress(data[, max_length]) -- Return a string containing the -## decompressed version of the data. - -## After calling this function, some of the input data may still be stored -## in internal buffers for later processing. - -## Call the flush() method to clear these buffers. - -## If the max_length parameter is specified then the return value will be -## no longer than max_length. Unconsumed input data will be stored in the -## unconsumed_tail attribute. -## """ -## return self.space.wrap('') -## decompress.unwrap_spec = ['self', str, int] - - -## def flush(self, length=0): -## """ -## flush( [length] ) -- Return a string containing any remaining -## decompressed data. length, if given, is the initial size of the output -## buffer. - -## The decompressor object can no longer be used after this call. -## """ -## return self.space.wrap('') -## flush.unwrap_spec = ['self', int] - - -##def Decompress___new__(space, w_subtype, w_anything=None): -## """ -## Create a new Decompress and call its initializer. -## """ -## stream = space.allocate_instance(Decompress, w_subtype) -## stream = space.interp_w(Decompress, stream) -## Decompress.__init__(stream, space, w_anything) -## return space.wrap(stream) -##Decompress___new__.unwrap_spec = [ObjSpace, W_Root, W_Root] - - -##Decompress.typedef = TypeDef( -## 'Decompress', -## __new__ = interp2app(Decompress___new__), -## decompress = interp2app(Decompress.decompress), -## flush = interp2app(Decompress.flush)) +def compress(stream, data, flush=Z_NO_FLUSH): + """ + Feed more data into a deflate stream. Returns a string containing + (a part of) the compressed data. If flush != Z_NO_FLUSH, this also + flushes the output data; see zlib.h or the documentation of the + zlib module for the possible values of 'flush'. + """ + # Warning, reentrant calls to the zlib with a given stream can cause it + # to crash. The caller of pypy.rlib.rzlib should use locks if needed. + + # Prepare the input buffer for the stream + inbuf = lltype.malloc(rffi.CCHARP.TO, len(data), flavor='raw') + try: + for i in xrange(len(data)): + inbuf[i] = data[i] + stream.c_next_in = rffi.cast(Bytefp, inbuf) + rffi.setintfield(stream, 'c_avail_in', len(data)) + + # Prepare the output buffer + outbuf = lltype.malloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE, + flavor='raw') + try: + # Strategy: we call deflate() to get as much output data as + # fits in the buffer, then accumulate all output into a list + # of characters 'result'. We don't need to gradually + # increase the output buffer size because there is no + # quadratic factor. + result = [] + + while True: + stream.c_next_out = rffi.cast(Bytefp, outbuf) + rffi.setintfield(stream, 'c_avail_out', OUTPUT_BUFFER_SIZE) + err = _deflate(stream, flush) + if err == Z_OK or err == Z_STREAM_END: + # accumulate data into 'result' + avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) + for i in xrange(OUTPUT_BUFFER_SIZE - avail_out): + result.append(outbuf[i]) + # if the output buffer is full, there might be more data + # so we need to try again. Otherwise, we're done. + if avail_out > 0: + break + # We're also done if we got a Z_STREAM_END (which should + # only occur when flush == Z_FINISH). + if err == Z_STREAM_END: + break + elif err == Z_BUF_ERROR: + # We will only get Z_BUF_ERROR if the output buffer + # was full but there wasn't more output when we + # tried again, so it is not an error condition. + avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) + assert avail_out == OUTPUT_BUFFER_SIZE + break + else: + raise RZlibError.fromstream(stream, err, + "while compressing") + finally: + lltype.free(outbuf, flavor='raw') + finally: + lltype.free(inbuf, flavor='raw') + + assert not stream.c_avail_in, "not all input consumed by deflate()" + return ''.join(result) Modified: pypy/dist/pypy/rlib/test/test_rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rzlib.py (original) +++ pypy/dist/pypy/rlib/test/test_rzlib.py Tue Sep 25 18:25:40 2007 @@ -81,15 +81,16 @@ rzlib.deflateEnd(stream) -##def test_compression(self): -## """ -## zlib.compressobj should return an object which can be used to compress -## bytes. -## """ -## compressor = rzlib.compressobj() -## bytes = compressor.compress(self.expanded) -## bytes += compressor.flush() -## assert bytes == self.compressed +def test_compression(): + """ + Once we have got a deflate stream, rzlib.compress() and rzlib.flush() + should allow us to compress bytes. + """ + stream = rzlib.deflateInit() + bytes = rzlib.compress(stream, expanded) + bytes += rzlib.compress(stream, "", rzlib.Z_FINISH) + rzlib.deflateEnd(stream) + assert bytes == compressed ##def test_decompression(self): From arigo at codespeak.net Tue Sep 25 18:30:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 18:30:16 +0200 (CEST) Subject: [pypy-svn] r46865 - pypy/dist/pypy/rlib/test Message-ID: <20070925163016.64A3E815D@code0.codespeak.net> Author: arigo Date: Tue Sep 25 18:30:15 2007 New Revision: 46865 Modified: pypy/dist/pypy/rlib/test/test_rzlib.py Log: Another test. Modified: pypy/dist/pypy/rlib/test/test_rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rzlib.py (original) +++ pypy/dist/pypy/rlib/test/test_rzlib.py Tue Sep 25 18:30:15 2007 @@ -93,6 +93,19 @@ assert bytes == compressed +def test_compression_lots_of_data(): + """ + Test compression of more data that fits in a single internal output buffer. + """ + expanded = repr(range(20000)) + compressed = zlib.compress(expanded) + print len(expanded), '=>', len(compressed) + stream = rzlib.deflateInit() + bytes = rzlib.compress(stream, expanded, rzlib.Z_FINISH) + rzlib.deflateEnd(stream) + assert bytes == compressed + + ##def test_decompression(self): ## """ ## zlib.decompressobj should return an object which can be used to From arigo at codespeak.net Tue Sep 25 19:04:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 19:04:06 +0200 (CEST) Subject: [pypy-svn] r46866 - in pypy/dist/pypy/rlib: . test Message-ID: <20070925170406.1C0BF8160@code0.codespeak.net> Author: arigo Date: Tue Sep 25 19:04:06 2007 New Revision: 46866 Modified: pypy/dist/pypy/rlib/rzlib.py pypy/dist/pypy/rlib/test/test_rzlib.py Log: Decompression. We can share most of the code with compression. Modified: pypy/dist/pypy/rlib/rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/rzlib.py (original) +++ pypy/dist/pypy/rlib/rzlib.py Tue Sep 25 19:04:06 2007 @@ -7,7 +7,7 @@ constantnames = ''' Z_OK Z_STREAM_ERROR Z_BUF_ERROR Z_MEM_ERROR Z_STREAM_END Z_DEFLATED Z_DEFAULT_STRATEGY Z_DEFAULT_COMPRESSION - Z_NO_FLUSH Z_FINISH + Z_NO_FLUSH Z_FINISH Z_SYNC_FLUSH MAX_WBITS MAX_MEM_LEVEL '''.split() @@ -117,6 +117,23 @@ stream, level, method, wbits, memlevel, strategy, ZLIB_VERSION, size) return result +# XXX I also want to call inflateInit2 instead of inflateInit2_ +_inflateInit2_ = zlib_external( + 'inflateInit2_', + [z_stream_p, # stream + rffi.INT, # window bits + rffi.CCHARP, # version + rffi.INT], # stream size + rffi.INT) +_inflate = zlib_external('inflate', [z_stream_p, rffi.INT], rffi.INT) + +_inflateEnd = zlib_external('inflateEnd', [z_stream_p], rffi.INT) + +def _inflateInit2(stream, wbits): + size = rffi.sizeof(z_stream) + result = _inflateInit2_(stream, wbits, ZLIB_VERSION, size) + return result + # ____________________________________________________________ CRC32_DEFAULT_START = 0 @@ -209,6 +226,40 @@ lltype.free(stream, flavor='raw') +def inflateInit(wbits=MAX_WBITS): + """ + Allocate and return an opaque 'stream' object that can be used to + decompress data. + """ + stream = lltype.malloc(z_stream, flavor='raw', zero=True) + err = _inflateInit2(stream, wbits) + if err == Z_OK: + return stream + else: + try: + if err == Z_STREAM_ERROR: + raise ValueError("Invalid initialization option") + else: + raise RZlibError.fromstream(stream, err, + "while creating decompression object") + finally: + lltype.free(stream, flavor='raw') + + +def inflateEnd(stream): + """ + Free the resources associated with the inflate stream. + Note that this may raise RZlibError. + """ + try: + err = _inflateEnd(stream) + if err != Z_OK: + raise RZlibError.fromstream(stream, err, + "while finishing decompression") + finally: + lltype.free(stream, flavor='raw') + + def compress(stream, data, flush=Z_NO_FLUSH): """ Feed more data into a deflate stream. Returns a string containing @@ -218,7 +269,30 @@ """ # Warning, reentrant calls to the zlib with a given stream can cause it # to crash. The caller of pypy.rlib.rzlib should use locks if needed. + return _operate(stream, data, flush, _deflate, "while compressing") + +def decompress(stream, data, flush=Z_NO_FLUSH): + """ + Feed more data into an inflate stream. Returns a string containing + (a part of) the decompressed data. If flush != Z_NO_FLUSH, this also + flushes the output data; see zlib.h or the documentation of the + zlib module for the possible values of 'flush'. + """ + # Warning, reentrant calls to the zlib with a given stream can cause it + # to crash. The caller of pypy.rlib.rzlib should use locks if needed. + + # _operate() does not support the Z_FINISH method of decompressing, + # which is for one-shot decompression where a single output buffer is + # large enough. We can just map Z_FINISH to Z_SYNC_FLUSH. + if flush == Z_FINISH: + flush = Z_SYNC_FLUSH + return _operate(stream, data, flush, _inflate, "while decompressing") + + +def _operate(stream, data, flush, cfunc, while_doing): + """Common code for compress() and decompress(). + """ # Prepare the input buffer for the stream inbuf = lltype.malloc(rffi.CCHARP.TO, len(data), flavor='raw') try: @@ -241,7 +315,7 @@ while True: stream.c_next_out = rffi.cast(Bytefp, outbuf) rffi.setintfield(stream, 'c_avail_out', OUTPUT_BUFFER_SIZE) - err = _deflate(stream, flush) + err = cfunc(stream, flush) if err == Z_OK or err == Z_STREAM_END: # accumulate data into 'result' avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) @@ -255,20 +329,24 @@ # only occur when flush == Z_FINISH). if err == Z_STREAM_END: break + else: + continue elif err == Z_BUF_ERROR: - # We will only get Z_BUF_ERROR if the output buffer - # was full but there wasn't more output when we - # tried again, so it is not an error condition. avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) - assert avail_out == OUTPUT_BUFFER_SIZE - break - else: - raise RZlibError.fromstream(stream, err, - "while compressing") + # When compressing, we will only get Z_BUF_ERROR if + # the output buffer was full but there wasn't more + # output when we tried again, so it is not an error + # condition. + if avail_out == OUTPUT_BUFFER_SIZE: + break + + # fallback case: report this error + raise RZlibError.fromstream(stream, err, while_doing) + finally: lltype.free(outbuf, flavor='raw') finally: lltype.free(inbuf, flavor='raw') - assert not stream.c_avail_in, "not all input consumed by deflate()" + assert not stream.c_avail_in, "not all input consumed by deflate/inflate" return ''.join(result) Modified: pypy/dist/pypy/rlib/test/test_rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rzlib.py (original) +++ pypy/dist/pypy/rlib/test/test_rzlib.py Tue Sep 25 19:04:06 2007 @@ -73,7 +73,7 @@ raises(ValueError, rzlib.deflateInit, 10) -def test_init_end(): +def test_deflate_init_end(): """ deflateInit() followed by deflateEnd() should work and do nothing. """ @@ -83,7 +83,7 @@ def test_compression(): """ - Once we have got a deflate stream, rzlib.compress() and rzlib.flush() + Once we have got a deflate stream, rzlib.compress() should allow us to compress bytes. """ stream = rzlib.deflateInit() @@ -106,12 +106,34 @@ assert bytes == compressed -##def test_decompression(self): -## """ -## zlib.decompressobj should return an object which can be used to -## decompress bytes. -## """ -## decompressor = self.zlib.decompressobj() -## bytes = decompressor.decompress(self.compressed) -## bytes += decompressor.flush() -## assert bytes == self.expanded +def test_inflate_init_end(): + """ + inflateInit() followed by inflateEnd() should work and do nothing. + """ + stream = rzlib.inflateInit() + rzlib.inflateEnd(stream) + + +def test_decompression(): + """ + Once we have got a inflate stream, rzlib.decompress() + should allow us to decompress bytes. + """ + stream = rzlib.inflateInit() + bytes = rzlib.decompress(stream, compressed) + bytes += rzlib.decompress(stream, "", rzlib.Z_FINISH) + rzlib.inflateEnd(stream) + assert bytes == expanded + + +def test_decompression_lots_of_data(): + """ + Test compression of more data that fits in a single internal output buffer. + """ + expanded = repr(range(20000)) + compressed = zlib.compress(expanded) + print len(compressed), '=>', len(expanded) + stream = rzlib.inflateInit() + bytes = rzlib.decompress(stream, compressed, rzlib.Z_FINISH) + rzlib.inflateEnd(stream) + assert bytes == expanded From arigo at codespeak.net Tue Sep 25 19:06:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 19:06:45 +0200 (CEST) Subject: [pypy-svn] r46867 - pypy/dist/pypy/module/zlib Message-ID: <20070925170645.68E3A8161@code0.codespeak.net> Author: arigo Date: Tue Sep 25 19:06:45 2007 New Revision: 46867 Added: pypy/dist/pypy/module/zlib/ - copied from r46866, pypy/branch/zlib-rffi-impl/pypy/module/zlib/ Log: Copy pypy.module.zlib from the branch (credits exarkun). From arigo at codespeak.net Tue Sep 25 19:42:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 19:42:39 +0200 (CEST) Subject: [pypy-svn] r46868 - in pypy/dist/pypy/rlib: . test Message-ID: <20070925174239.0A1FB8137@code0.codespeak.net> Author: arigo Date: Tue Sep 25 19:42:39 2007 New Revision: 46868 Modified: pypy/dist/pypy/rlib/rzlib.py pypy/dist/pypy/rlib/test/test_rzlib.py Log: Fix the decompression flushing, now that I understood the purpose of the flush() method of decompression objects: it's there only to complain if the input stream was truncated at some point. Modified: pypy/dist/pypy/rlib/rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/rzlib.py (original) +++ pypy/dist/pypy/rlib/rzlib.py Tue Sep 25 19:42:39 2007 @@ -189,6 +189,8 @@ return RZlibError(msg) fromstream = staticmethod(fromstream) +null_stream = lltype.nullptr(z_stream) + def deflateInit(level=Z_DEFAULT_COMPRESSION, method=Z_DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, @@ -215,15 +217,9 @@ def deflateEnd(stream): """ Free the resources associated with the deflate stream. - Note that this may raise RZlibError. """ - try: - err = _deflateEnd(stream) - if err != Z_OK: - raise RZlibError.fromstream(stream, err, - "while finishing compression") - finally: - lltype.free(stream, flavor='raw') + _deflateEnd(stream) + lltype.free(stream, flavor='raw') def inflateInit(wbits=MAX_WBITS): @@ -251,13 +247,8 @@ Free the resources associated with the inflate stream. Note that this may raise RZlibError. """ - try: - err = _inflateEnd(stream) - if err != Z_OK: - raise RZlibError.fromstream(stream, err, - "while finishing decompression") - finally: - lltype.free(stream, flavor='raw') + _inflateEnd(stream) + lltype.free(stream, flavor='raw') def compress(stream, data, flush=Z_NO_FLUSH): @@ -269,10 +260,10 @@ """ # Warning, reentrant calls to the zlib with a given stream can cause it # to crash. The caller of pypy.rlib.rzlib should use locks if needed. - return _operate(stream, data, flush, _deflate, "while compressing") + return _operate(stream, data, flush, False, _deflate, "while compressing") -def decompress(stream, data, flush=Z_NO_FLUSH): +def decompress(stream, data, flush=Z_SYNC_FLUSH): """ Feed more data into an inflate stream. Returns a string containing (a part of) the decompressed data. If flush != Z_NO_FLUSH, this also @@ -282,15 +273,19 @@ # Warning, reentrant calls to the zlib with a given stream can cause it # to crash. The caller of pypy.rlib.rzlib should use locks if needed. - # _operate() does not support the Z_FINISH method of decompressing, - # which is for one-shot decompression where a single output buffer is - # large enough. We can just map Z_FINISH to Z_SYNC_FLUSH. + # _operate() does not support the Z_FINISH method of decompressing. + # We can use Z_SYNC_FLUSH instead and manually check that we got to + # the end of the data. if flush == Z_FINISH: flush = Z_SYNC_FLUSH - return _operate(stream, data, flush, _inflate, "while decompressing") + should_finish = True + else: + should_finish = False + return _operate(stream, data, flush, should_finish, _inflate, + "while decompressing") -def _operate(stream, data, flush, cfunc, while_doing): +def _operate(stream, data, flush, should_finish, cfunc, while_doing): """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream @@ -348,5 +343,11 @@ finally: lltype.free(inbuf, flavor='raw') + # When decompressing, if the compressed stream of data was truncated, + # then the zlib simply returns Z_OK and waits for more. Let's detect + # this situation and complain. + if should_finish and err != Z_STREAM_END: + raise RZlibError("the input compressed stream of data is not complete") + assert not stream.c_avail_in, "not all input consumed by deflate/inflate" return ''.join(result) Modified: pypy/dist/pypy/rlib/test/test_rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rzlib.py (original) +++ pypy/dist/pypy/rlib/test/test_rzlib.py Tue Sep 25 19:42:39 2007 @@ -3,6 +3,7 @@ Tests for the rzlib module. """ +import py from pypy.rlib import rzlib from pypy.rlib.rarithmetic import r_uint import zlib @@ -137,3 +138,22 @@ bytes = rzlib.decompress(stream, compressed, rzlib.Z_FINISH) rzlib.inflateEnd(stream) assert bytes == expanded + + +def test_decompression_truncated_input(): + """ + Test that we can accept incomplete input when inflating, but also + detect this situation when using Z_FINISH. + """ + expanded = repr(range(20000)) + compressed = zlib.compress(expanded) + print len(compressed), '=>', len(expanded) + stream = rzlib.inflateInit() + data = rzlib.decompress(stream, compressed[:1000]) + assert expanded.startswith(data) + data += rzlib.decompress(stream, compressed[1000:2000]) + assert expanded.startswith(data) + py.test.raises(rzlib.RZlibError, + rzlib.decompress, stream, compressed[2000:-500], + rzlib.Z_FINISH) + rzlib.inflateEnd(stream) From arigo at codespeak.net Tue Sep 25 19:51:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 19:51:40 +0200 (CEST) Subject: [pypy-svn] r46869 - in pypy/dist/pypy/module/zlib: . test Message-ID: <20070925175140.0AB148139@code0.codespeak.net> Author: arigo Date: Tue Sep 25 19:51:40 2007 New Revision: 46869 Modified: pypy/dist/pypy/module/zlib/__init__.py pypy/dist/pypy/module/zlib/app_zlib.py pypy/dist/pypy/module/zlib/interp_zlib.py pypy/dist/pypy/module/zlib/test/test_zlib.py Log: Make this code use pypy.rlib.rzlib. Modified: pypy/dist/pypy/module/zlib/__init__.py ============================================================================== --- pypy/dist/pypy/module/zlib/__init__.py (original) +++ pypy/dist/pypy/module/zlib/__init__.py Tue Sep 25 19:51:40 2007 @@ -10,14 +10,12 @@ interpleveldefs = { 'crc32': 'interp_zlib.crc32', 'adler32': 'interp_zlib.adler32', - 'Compress': 'interp_zlib.Compress', - 'Decompress': 'interp_zlib.Decompress', + 'compressobj': 'interp_zlib.Compress', + 'decompressobj': 'interp_zlib.Decompress', } appleveldefs = { 'error': 'app_zlib.error', 'compress': 'app_zlib.compress', 'decompress': 'app_zlib.decompress', - 'compressobj': 'app_zlib.compressobj', - 'decompressobj': 'app_zlib.decompressobj', } Modified: pypy/dist/pypy/module/zlib/app_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/app_zlib.py (original) +++ pypy/dist/pypy/module/zlib/app_zlib.py Tue Sep 25 19:51:40 2007 @@ -11,44 +11,27 @@ """ -def compressobj(level=None): - """ - compressobj([level]) -- Return a compressor object. - - Optional arg level is the compression level, in 1-9. - """ - import zlib - return zlib.Compress(level) - +# XXX the following should be moved to interp-level to avoid various overheads - -def decompressobj(wbits=None): - """ - decompressobj([wbits]) -- Return a decompressor object. - - Optional arg wbits is the window buffer size. - """ - import zlib - return zlib.Decompress(wbits) - - -def compress(string, level=None): +def compress(string, *args): """ compress(string[, level]) -- Returned compressed string. Optional arg level is the compression level, in 1-9. """ - compressor = compressobj(level) + import zlib + compressor = zlib.compressobj(*args) return compressor.compress(string) + compressor.flush() -def decompress(string, wbits=None, bufsize=None): +def decompress(string, *args): """ decompress(string[, wbits[, bufsize]]) -- Return decompressed string. Optional arg wbits is the window buffer size. Optional arg bufsize is the initial output buffer size. """ - # XXX bufsize is ignored because it's basically useless. - decompressor = decompressobj(wbits) + # XXX bufsize is not accepted by this version but it's basically useless. + import zlib + decompressor = zlib.decompressobj(*args) return decompressor.decompress(string) + decompressor.flush() Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Tue Sep 25 19:51:40 2007 @@ -1,182 +1,49 @@ -# import zlib - from pypy.interpreter.gateway import ObjSpace, W_Root, interp2app from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.error import OperationError -from pypy.rpython.lltypesystem import rffi, lltype, llmemory -from pypy.rpython.tool import rffi_platform - -includes = ['zlib.h'] -libraries = ['z'] - -class SimpleCConfig: - """ - Definitions for basic types defined by zlib. - """ - _includes_ = includes - - # XXX If Z_PREFIX was defined for the libz build, then these types are - # named z_uInt, z_uLong, and z_Bytef instead. - uInt = rffi_platform.SimpleType('uInt', rffi.UINT) - uLong = rffi_platform.SimpleType('uLong', rffi.ULONG) - Bytef = rffi_platform.SimpleType('Bytef', rffi.UCHAR) - voidpf = rffi_platform.SimpleType('voidpf', rffi.VOIDP) - - Z_OK = rffi_platform.ConstantInteger('Z_OK') - Z_STREAM_ERROR = rffi_platform.ConstantInteger('Z_STREAM_ERROR') - - ZLIB_VERSION = rffi_platform.DefinedConstantString('ZLIB_VERSION') - - Z_DEFLATED = rffi_platform.ConstantInteger('Z_DEFLATED') - Z_DEFAULT_STRATEGY = rffi_platform.ConstantInteger('Z_DEFAULT_STRATEGY') - Z_DEFAULT_COMPRESSION = rffi_platform.ConstantInteger( - 'Z_DEFAULT_COMPRESSION') - Z_NO_FLUSH = rffi_platform.ConstantInteger( - 'Z_NO_FLUSH') - -config = rffi_platform.configure(SimpleCConfig) -voidpf = config['voidpf'] -uInt = config['uInt'] -uLong = config['uLong'] -Bytef = config['Bytef'] -Bytefp = lltype.Ptr(lltype.Array(Bytef, hints={'nolength': True})) - -Z_OK = config['Z_OK'] -Z_STREAM_ERROR = config['Z_STREAM_ERROR'] - -ZLIB_VERSION = config['ZLIB_VERSION'] - -Z_DEFAULT_COMPRESSION = config['Z_DEFAULT_COMPRESSION'] -Z_DEFAULT_STRATEGY = config['Z_DEFAULT_STRATEGY'] -Z_NO_FLUSH = config['Z_DEFAULT_COMPRESSION'] -Z_DEFLATED = config['Z_DEFLATED'] - -class ComplexCConfig: - """ - Definitions of structure types defined by zlib and based on SimpleCConfig - definitions. - """ - _includes_ = includes - - z_stream = rffi_platform.Struct( - 'z_stream', - [('next_in', Bytefp), - ('avail_in', uInt), - ('total_in', uLong), - - ('next_out', Bytefp), - ('avail_out', uInt), - ('total_out', uLong), - - ('msg', rffi.CCHARP), - - ('zalloc', lltype.Ptr( - lltype.FuncType([voidpf, uInt, uInt], voidpf))), - ('zfree', lltype.Ptr( - lltype.FuncType([voidpf, voidpf], lltype.Void))), - - ('opaque', voidpf), - - ('data_type', rffi.INT), - ('adler', uLong), - ('reserved', uLong) - ]) - -config = rffi_platform.configure(ComplexCConfig) -z_stream = config['z_stream'] -z_stream_p = lltype.Ptr(z_stream) - -def zlib_external(*a, **kw): - kw['includes'] = includes - kw['libraries'] = libraries - return rffi.llexternal(*a, **kw) - -_crc32 = zlib_external('crc32', [uLong, Bytefp, uInt], uLong) -_adler32 = zlib_external('adler32', [uLong, Bytefp, uInt], uLong) - - -# XXX I want to call deflateInit2, not deflateInit2_ -_deflateInit2_ = zlib_external( - 'deflateInit2_', - [z_stream_p, # stream - rffi.INT, # level - rffi.INT, # method - rffi.INT, # window bits - rffi.INT, # mem level - rffi.INT, # strategy - rffi.CCHARP, # version - rffi.INT], # stream size - rffi.INT) -_deflate = zlib_external('deflate', [z_stream_p, rffi.INT], rffi.INT) - -def _deflateInit2(stream, level, method, wbits, memlevel, strategy): - version = rffi.str2charp(ZLIB_VERSION) - size = llmemory.sizeof(z_stream) - result = _deflateInit2_( - stream, level, method, wbits, memlevel, strategy, version, size) - rffi.free_charp(version) - return result +from pypy.rlib import rzlib -def crc32(space, string, start=0): +def crc32(space, string, start = rzlib.CRC32_DEFAULT_START): """ crc32(string[, start]) -- Compute a CRC-32 checksum of string. An optional starting value can be specified. The returned checksum is an integer. """ - bytes = rffi.str2charp(string) - checksum = _crc32(start, rffi.cast(Bytefp, bytes), len(string)) - rffi.free_charp(bytes) + checksum = rzlib.crc32(string, start) # This is, perhaps, a little stupid. zlib returns the checksum unsigned. # CPython exposes it as a signed value, though. -exarkun - checksum = rffi.cast(rffi.INT, checksum) + # The value *is* unsigned on 64-bit platforms in CPython... bah. + # For now let's do the same as CPython and boldly cast to a C long. -arigo + checksum = int(checksum) return space.wrap(checksum) crc32.unwrap_spec = [ObjSpace, str, int] -def adler32(space, string, start=1): +def adler32(space, string, start = rzlib.ADLER32_DEFAULT_START): """ adler32(string[, start]) -- Compute an Adler-32 checksum of string. An optional starting value can be specified. The returned checksum is an integer. """ - bytes = rffi.str2charp(string) - checksum = _adler32(start, rffi.cast(Bytefp, bytes), len(string)) - rffi.free_charp(bytes) + checksum = rzlib.adler32(string, start) # This is, perhaps, a little stupid. zlib returns the checksum unsigned. # CPython exposes it as a signed value, though. -exarkun - checksum = rffi.cast(rffi.INT, checksum) + # The value *is* unsigned on 64-bit platforms in CPython... bah. + # For now let's do the same as CPython and boldly cast to a C long. -arigo + checksum = int(checksum) return space.wrap(checksum) adler32.unwrap_spec = [ObjSpace, str, int] -class _StreamBase(Wrappable): - """ - Base for classes which want to have a z_stream allocated when they are - initialized and de-allocated when they are freed. - """ - def __init__(self, space): - self.space = space - self.stream = lltype.malloc(z_stream, flavor='raw') - self.stream.c_zalloc = lltype.nullptr(z_stream.c_zalloc.TO) - self.stream.c_zfree = lltype.nullptr(z_stream.c_zfree.TO) - self.stream.c_avail_in = rffi.cast(lltype.Unsigned, 0) - self.stream.c_next_in = lltype.nullptr(z_stream.c_next_in.TO) - - - def __del__(self): - lltype.free(self.stream, flavor='raw') - - - def error_from_zlib(space, status): if status == Z_STREAM_ERROR: return OperationError( @@ -185,34 +52,33 @@ assert False, "unhandled status %s" % (status,) -class Compress(_StreamBase): +class Compress(Wrappable): """ Wrapper around zlib's z_stream structure which provides convenient compression functionality. """ - def __init__(self, space, w_level): + stream = rzlib.null_stream + + def __init__(self, space, level=rzlib.Z_DEFAULT_COMPRESSION): # XXX CPython actually exposes 4 more undocumented parameters beyond # level. - if space.is_w(w_level, space.w_None): - level = Z_DEFAULT_COMPRESSION - else: - level = space.int_w(w_level) - - _StreamBase.__init__(self, space) - - method = Z_DEFLATED - windowBits = 15 - memLevel = 8 - strategy = Z_DEFAULT_STRATEGY - - result = _deflateInit2( - self.stream, level, method, windowBits, memLevel, strategy) + self.space = space + try: + self.stream = rzlib.deflateInit(level) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("Invalid initialization option")) - if result != Z_OK: - raise error_from_zlib(self.space, result) + def __del__(self): + """Automatically free the resources used by the stream.""" + if self.stream: + rzlib.deflateEnd(self.stream) + self.stream = null_stream - def compress(self, data, length=16384): + def compress(self, data): """ compress(data) -- Return a string containing data compressed. @@ -221,22 +87,15 @@ Call the flush() method to clear these buffers. """ - self.stream.c_avail_in = rffi.cast(lltype.Unsigned, len(data)) - self.stream.c_next_in = lltype.malloc( - Bytefp.TO, len(data), flavor='raw') - for i in xrange(len(data)): - self.stream.c_next_in[i] = rffi.cast(Bytef, data[i]) - self.stream.c_avail_out = rffi.cast(lltype.Unsigned, length) - self.stream.c_next_out = lltype.malloc( - Bytefp.TO, length, flavor='raw') - result = _deflate(self.stream, Z_NO_FLUSH) - if result != Z_OK: - raise error_from_zlib(self.space, result) - return rffi.charp2str(self.stream.c_next_out) - compress.unwrap_spec = ['self', str, int] + try: + result = rzlib.compress(self.stream, data) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + return self.space.wrap(result) + compress.unwrap_spec = ['self', str] - def flush(self, mode=0): # XXX =Z_FINISH + def flush(self, mode=rzlib.Z_FINISH): """ flush( [mode] ) -- Return a string containing any remaining compressed data. @@ -248,19 +107,23 @@ calling the flush() method. Otherwise, more data can still be compressed. """ - return self.space.wrap('') + try: + result = rzlib.compress(self.stream, '', mode) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + return self.space.wrap(result) flush.unwrap_spec = ['self', int] -def Compress___new__(space, w_subtype, w_level=None): +def Compress___new__(space, w_subtype, level=rzlib.Z_DEFAULT_COMPRESSION): """ Create a new z_stream and call its initializer. """ stream = space.allocate_instance(Compress, w_subtype) stream = space.interp_w(Compress, stream) - Compress.__init__(stream, space, w_level) + Compress.__init__(stream, space, level) return space.wrap(stream) -Compress___new__.unwrap_spec = [ObjSpace, W_Root, W_Root] +Compress___new__.unwrap_spec = [ObjSpace, W_Root, int] Compress.typedef = TypeDef( @@ -271,12 +134,14 @@ -class Decompress(_StreamBase): +class Decompress(Wrappable): """ Wrapper around zlib's z_stream structure which provides convenient decompression functionality. """ - def __init__(self, space, wbits): + stream = rzlib.null_stream + + def __init__(self, space, wbits=rzlib.MAX_WBITS): """ Initialize a new decompression object. @@ -285,11 +150,20 @@ and decompression. See the documentation for deflateInit2 and inflateInit2. """ - _StreamBase.__init__(self, space) - self.wbits = wbits + self.space = space + try: + self.stream = rzlib.inflateInit(wbits) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + + def __del__(self): + """Automatically free the resources used by the stream.""" + if self.stream: + rzlib.inflateEnd(self.stream) + self.stream = null_stream - def decompress(self, data, max_length=0): # XXX =None + def decompress(self, data, max_length=0): """ decompress(data[, max_length]) -- Return a string containing the decompressed version of the data. @@ -303,7 +177,14 @@ no longer than max_length. Unconsumed input data will be stored in the unconsumed_tail attribute. """ - return self.space.wrap('') + if max_length != 0: # XXX + raise OperationError(self.space.w_NotImplementedError, + self.space.wrap("max_length != 0")) + try: + result = rzlib.decompress(self.stream, data) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + return self.space.wrap(result) decompress.unwrap_spec = ['self', str, int] @@ -315,19 +196,23 @@ The decompressor object can no longer be used after this call. """ - return self.space.wrap('') + try: + result = rzlib.decompress(self.stream, '', rzlib.Z_FINISH) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + return self.space.wrap(result) flush.unwrap_spec = ['self', int] -def Decompress___new__(space, w_subtype, w_anything=None): +def Decompress___new__(space, w_subtype, wbits=rzlib.MAX_WBITS): """ Create a new Decompress and call its initializer. """ stream = space.allocate_instance(Decompress, w_subtype) stream = space.interp_w(Decompress, stream) - Decompress.__init__(stream, space, w_anything) + Decompress.__init__(stream, space, wbits) return space.wrap(stream) -Decompress___new__.unwrap_spec = [ObjSpace, W_Root, W_Root] +Decompress___new__.unwrap_spec = [ObjSpace, W_Root, int] Decompress.typedef = TypeDef( Modified: pypy/dist/pypy/module/zlib/test/test_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/test/test_zlib.py (original) +++ pypy/dist/pypy/module/zlib/test/test_zlib.py Tue Sep 25 19:51:40 2007 @@ -113,3 +113,19 @@ bytes = decompressor.decompress(self.compressed) bytes += decompressor.flush() assert bytes == self.expanded + + + def test_compress(self): + """ + Test the zlib.compress() function. + """ + bytes = self.zlib.compress(self.expanded) + assert bytes == self.compressed + + + def test_decompress(self): + """ + Test the zlib.decompress() function. + """ + bytes = self.zlib.decompress(self.compressed) + assert bytes == self.expanded From arigo at codespeak.net Tue Sep 25 19:57:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 19:57:17 +0200 (CEST) Subject: [pypy-svn] r46870 - pypy/dist/pypy/module/zlib Message-ID: <20070925175717.14E8F8139@code0.codespeak.net> Author: arigo Date: Tue Sep 25 19:57:16 2007 New Revision: 46870 Modified: pypy/dist/pypy/module/zlib/__init__.py pypy/dist/pypy/module/zlib/app_zlib.py pypy/dist/pypy/module/zlib/interp_zlib.py Log: Move the compress() and decompress() function at interp-level. Modified: pypy/dist/pypy/module/zlib/__init__.py ============================================================================== --- pypy/dist/pypy/module/zlib/__init__.py (original) +++ pypy/dist/pypy/module/zlib/__init__.py Tue Sep 25 19:57:16 2007 @@ -12,10 +12,10 @@ 'adler32': 'interp_zlib.adler32', 'compressobj': 'interp_zlib.Compress', 'decompressobj': 'interp_zlib.Decompress', + 'compress': 'interp_zlib.compress', + 'decompress': 'interp_zlib.decompress', } appleveldefs = { 'error': 'app_zlib.error', - 'compress': 'app_zlib.compress', - 'decompress': 'app_zlib.decompress', } Modified: pypy/dist/pypy/module/zlib/app_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/app_zlib.py (original) +++ pypy/dist/pypy/module/zlib/app_zlib.py Tue Sep 25 19:57:16 2007 @@ -9,29 +9,3 @@ """ Raised by zlib operations. """ - - -# XXX the following should be moved to interp-level to avoid various overheads - -def compress(string, *args): - """ - compress(string[, level]) -- Returned compressed string. - - Optional arg level is the compression level, in 1-9. - """ - import zlib - compressor = zlib.compressobj(*args) - return compressor.compress(string) + compressor.flush() - - -def decompress(string, *args): - """ - decompress(string[, wbits[, bufsize]]) -- Return decompressed string. - - Optional arg wbits is the window buffer size. Optional arg bufsize is - the initial output buffer size. - """ - # XXX bufsize is not accepted by this version but it's basically useless. - import zlib - decompressor = zlib.decompressobj(*args) - return decompressor.decompress(string) + decompressor.flush() Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Tue Sep 25 19:57:16 2007 @@ -44,12 +44,49 @@ adler32.unwrap_spec = [ObjSpace, str, int] -def error_from_zlib(space, status): - if status == Z_STREAM_ERROR: - return OperationError( - space.w_ValueError, - space.wrap("Invalid initialization option")) - assert False, "unhandled status %s" % (status,) +def compress(space, string, level=rzlib.Z_DEFAULT_COMPRESSION): + """ + compress(string[, level]) -- Returned compressed string. + + Optional arg level is the compression level, in 1-9. + """ + try: + try: + stream = rzlib.deflateInit(level) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("Invalid initialization option")) + try: + result = rzlib.compress(stream, string, rzlib.Z_FINISH) + finally: + rzlib.deflateEnd(stream) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + return space.wrap(result) +compress.unwrap_spec = [ObjSpace, str, int] + + +def decompress(space, string, wbits=rzlib.MAX_WBITS, bufsize=0): + """ + decompress(string[, wbits[, bufsize]]) -- Return decompressed string. + + Optional arg wbits is the window buffer size. Optional arg bufsize is + the initial output buffer size. + """ + try: + try: + stream = rzlib.inflateInit(wbits) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("Invalid initialization option")) + try: + result = rzlib.decompress(stream, string, rzlib.Z_FINISH) + finally: + rzlib.inflateEnd(stream) + except rzlib.RZlibError, e: + raise zlib_error(self.space, e.msg) + return space.wrap(result) +decompress.unwrap_spec = [ObjSpace, str, int, int] class Compress(Wrappable): @@ -155,6 +192,9 @@ self.stream = rzlib.inflateInit(wbits) except rzlib.RZlibError, e: raise zlib_error(self.space, e.msg) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("Invalid initialization option")) def __del__(self): """Automatically free the resources used by the stream.""" From tismer at codespeak.net Tue Sep 25 20:01:08 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Tue, 25 Sep 2007 20:01:08 +0200 (CEST) Subject: [pypy-svn] r46871 - pypy/dist/pypy/module/_stackless Message-ID: <20070925180108.5C6808139@code0.codespeak.net> Author: tismer Date: Tue Sep 25 20:01:05 2007 New Revision: 46871 Added: pypy/dist/pypy/module/_stackless/interp_clonable.py - copied, changed from r46862, pypy/dist/pypy/module/_stackless/clonable.py pypy/dist/pypy/module/_stackless/rclonable.py - copied unchanged from r46862, pypy/dist/pypy/module/_stackless/interp_clonable.py Removed: pypy/dist/pypy/module/_stackless/clonable.py Modified: pypy/dist/pypy/module/_stackless/__init__.py Log: started renaming _stackless contents to conform with the standard. testing clonable was a major effort (needed to build) Modified: pypy/dist/pypy/module/_stackless/__init__.py ============================================================================== --- pypy/dist/pypy/module/_stackless/__init__.py (original) +++ pypy/dist/pypy/module/_stackless/__init__.py Tue Sep 25 20:01:05 2007 @@ -28,7 +28,7 @@ post_install_greenlet(self) if self.space.config.translation.gc in ('framework', 'stacklessgc'): - from pypy.module._stackless.clonable import post_install as post_install_clonable - self.extra_interpdef('clonable', 'clonable.AppClonableCoroutine') - self.extra_interpdef('fork', 'clonable.fork') + from pypy.module._stackless.interp_clonable import post_install as post_install_clonable + self.extra_interpdef('clonable', 'interp_clonable.AppClonableCoroutine') + self.extra_interpdef('fork', 'interp_clonable.fork') post_install_clonable(self) From arigo at codespeak.net Tue Sep 25 20:03:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 20:03:49 +0200 (CEST) Subject: [pypy-svn] r46872 - in pypy/dist/pypy/module/zlib: . test Message-ID: <20070925180349.9724F8139@code0.codespeak.net> Author: arigo Date: Tue Sep 25 20:03:48 2007 New Revision: 46872 Modified: pypy/dist/pypy/module/zlib/interp_zlib.py pypy/dist/pypy/module/zlib/test/test_zlib.py Log: What is not tested is broken. Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Tue Sep 25 20:03:48 2007 @@ -44,6 +44,12 @@ adler32.unwrap_spec = [ObjSpace, str, int] +def zlib_error(space, msg): + w_module = space.getbuiltinmodule('zlib') + w_error = space.getattr(w_module, space.wrap('error')) + return OperationError(w_error, space.wrap(msg)) + + def compress(space, string, level=rzlib.Z_DEFAULT_COMPRESSION): """ compress(string[, level]) -- Returned compressed string. @@ -61,7 +67,7 @@ finally: rzlib.deflateEnd(stream) except rzlib.RZlibError, e: - raise zlib_error(self.space, e.msg) + raise zlib_error(space, e.msg) return space.wrap(result) compress.unwrap_spec = [ObjSpace, str, int] @@ -84,7 +90,7 @@ finally: rzlib.inflateEnd(stream) except rzlib.RZlibError, e: - raise zlib_error(self.space, e.msg) + raise zlib_error(space, e.msg) return space.wrap(result) decompress.unwrap_spec = [ObjSpace, str, int, int] @@ -167,8 +173,11 @@ 'Compress', __new__ = interp2app(Compress___new__), compress = interp2app(Compress.compress), - flush = interp2app(Compress.flush)) + flush = interp2app(Compress.flush), + __doc__ = """compressobj([level]) -- Return a compressor object. +Optional arg level is the compression level, in 1-9. +""") class Decompress(Wrappable): @@ -259,4 +268,8 @@ 'Decompress', __new__ = interp2app(Decompress___new__), decompress = interp2app(Decompress.decompress), - flush = interp2app(Decompress.flush)) + flush = interp2app(Decompress.flush), + __doc__ = """decompressobj([wbits]) -- Return a decompressor object. + +Optional arg wbits is the window buffer size. +""") Modified: pypy/dist/pypy/module/zlib/test/test_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/test/test_zlib.py (original) +++ pypy/dist/pypy/module/zlib/test/test_zlib.py Tue Sep 25 20:03:48 2007 @@ -129,3 +129,8 @@ """ bytes = self.zlib.decompress(self.compressed) assert bytes == self.expanded + + + def test_decompress_invalid_input(self): + raises(self.zlib.error, self.zlib.decompress, self.compressed[:-2]) + raises(self.zlib.error, self.zlib.decompress, 'foobar') From arigo at codespeak.net Tue Sep 25 20:07:33 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 20:07:33 +0200 (CEST) Subject: [pypy-svn] r46873 - pypy/dist/pypy/module/zlib Message-ID: <20070925180733.24BEF813A@code0.codespeak.net> Author: arigo Date: Tue Sep 25 20:07:32 2007 New Revision: 46873 Modified: pypy/dist/pypy/module/zlib/interp_zlib.py Log: Typos found by running pypy/bin/checkmodule.py. Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Tue Sep 25 20:07:32 2007 @@ -2,6 +2,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.error import OperationError +from pypy.rlib.rarithmetic import intmask from pypy.rlib import rzlib @@ -19,7 +20,7 @@ # CPython exposes it as a signed value, though. -exarkun # The value *is* unsigned on 64-bit platforms in CPython... bah. # For now let's do the same as CPython and boldly cast to a C long. -arigo - checksum = int(checksum) + checksum = intmask(checksum) return space.wrap(checksum) crc32.unwrap_spec = [ObjSpace, str, int] @@ -38,7 +39,7 @@ # CPython exposes it as a signed value, though. -exarkun # The value *is* unsigned on 64-bit platforms in CPython... bah. # For now let's do the same as CPython and boldly cast to a C long. -arigo - checksum = int(checksum) + checksum = intmask(checksum) return space.wrap(checksum) adler32.unwrap_spec = [ObjSpace, str, int] @@ -118,7 +119,7 @@ """Automatically free the resources used by the stream.""" if self.stream: rzlib.deflateEnd(self.stream) - self.stream = null_stream + self.stream = rzlib.null_stream def compress(self, data): @@ -209,7 +210,7 @@ """Automatically free the resources used by the stream.""" if self.stream: rzlib.inflateEnd(self.stream) - self.stream = null_stream + self.stream = rzlib.null_stream def decompress(self, data, max_length=0): From arigo at codespeak.net Tue Sep 25 21:15:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 21:15:54 +0200 (CEST) Subject: [pypy-svn] r46876 - pypy/dist/pypy/rpython/tool Message-ID: <20070925191554.BBC268142@code0.codespeak.net> Author: arigo Date: Tue Sep 25 21:15:54 2007 New Revision: 46876 Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Trying to fix rffi_platform for Struct('name_without_a_struct_before'). Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Tue Sep 25 21:15:54 2007 @@ -252,12 +252,15 @@ seen[cell] = True name = self.name + hints = {'align': info['align'], + 'size': info['size'], + 'fieldoffsets': tuple(fieldoffsets), + 'padding': tuple(padfields)} if name.startswith('struct '): name = name[7:] - kwds = {'hints':{'align':info['align'], - 'size':info['size'], - 'fieldoffsets':tuple(fieldoffsets), - 'padding':tuple(padfields)}} + else: + hints['typedef'] = True + kwds = {'hints': hints} return rffi.CStruct(name, *fields, **kwds) class SimpleType(CConfigEntry): From tismer at codespeak.net Tue Sep 25 21:27:07 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Tue, 25 Sep 2007 21:27:07 +0200 (CEST) Subject: [pypy-svn] r46878 - in pypy/dist/pypy/module/_stackless: . test Message-ID: <20070925192707.742E68150@code0.codespeak.net> Author: tismer Date: Tue Sep 25 21:27:07 2007 New Revision: 46878 Added: pypy/dist/pypy/module/_stackless/interp_composable_coroutine - copied unchanged from r46862, pypy/dist/pypy/module/_stackless/composable_coroutine.py pypy/dist/pypy/module/_stackless/interp_composable_coroutine.py - copied, changed from r46862, pypy/dist/pypy/module/_stackless/composable_coroutine.py pypy/dist/pypy/module/_stackless/interp_coroutine.py - copied, changed from r46862, pypy/dist/pypy/module/_stackless/coroutine.py Removed: pypy/dist/pypy/module/_stackless/composable_coroutine.py pypy/dist/pypy/module/_stackless/coroutine.py Modified: pypy/dist/pypy/module/_stackless/__init__.py pypy/dist/pypy/module/_stackless/interp_clonable.py pypy/dist/pypy/module/_stackless/interp_greenlet.py pypy/dist/pypy/module/_stackless/test/test_choicepoint.py pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py pypy/dist/pypy/module/_stackless/test/test_interp_coroutine.py Log: renaming and reorganizing the _stackless folder. test renaming goes next, needed to check-in Modified: pypy/dist/pypy/module/_stackless/__init__.py ============================================================================== --- pypy/dist/pypy/module/_stackless/__init__.py (original) +++ pypy/dist/pypy/module/_stackless/__init__.py Tue Sep 25 21:27:07 2007 @@ -13,16 +13,16 @@ interpleveldefs = { 'tasklet' : 'interp_stackless.tasklet', - 'coroutine' : 'coroutine.AppCoroutine', + 'coroutine' : 'interp_coroutine.AppCoroutine', 'greenlet' : 'interp_greenlet.AppGreenlet', - 'usercostate': 'composable_coroutine.W_UserCoState', - '_return_main' : 'coroutine.return_main', + 'usercostate': 'interp_composable_coroutine.W_UserCoState', + '_return_main' : 'interp_coroutine.return_main', } def setup_after_space_initialization(self): # post-installing classmethods/staticmethods which # are not yet directly supported - from pypy.module._stackless.coroutine import post_install as post_install_coro + from pypy.module._stackless.interp_coroutine import post_install as post_install_coro post_install_coro(self) from pypy.module._stackless.interp_greenlet import post_install as post_install_greenlet post_install_greenlet(self) Modified: pypy/dist/pypy/module/_stackless/interp_clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/interp_clonable.py (original) +++ pypy/dist/pypy/module/_stackless/interp_clonable.py Tue Sep 25 21:27:07 2007 @@ -1,9 +1,9 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root -from pypy.module._stackless.coroutine import AppCoroutine, AppCoState -from pypy.module._stackless.coroutine import makeStaticMethod -from pypy.module._stackless.interp_coroutine import AbstractThunk +from pypy.module._stackless.interp_coroutine import AppCoroutine, AppCoState +from pypy.module._stackless.interp_coroutine import makeStaticMethod +from pypy.rlib.rcoroutine import AbstractThunk from pypy.module._stackless.rclonable import InterpClonableMixin Modified: pypy/dist/pypy/module/_stackless/interp_greenlet.py ============================================================================== --- pypy/dist/pypy/module/_stackless/interp_greenlet.py (original) +++ pypy/dist/pypy/module/_stackless/interp_greenlet.py Tue Sep 25 21:27:07 2007 @@ -4,9 +4,9 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.error import OperationError -from pypy.module._stackless.interp_coroutine import Coroutine, BaseCoState -from pypy.module._stackless.interp_coroutine import AbstractThunk, syncstate -from pypy.module._stackless.coroutine import makeStaticMethod +from pypy.rlib.rcoroutine import Coroutine, BaseCoState +from pypy.rlib.rcoroutine import AbstractThunk, syncstate +from pypy.module._stackless.interp_coroutine import makeStaticMethod class GreenletThunk(AbstractThunk): Modified: pypy/dist/pypy/module/_stackless/test/test_choicepoint.py ============================================================================== --- pypy/dist/pypy/module/_stackless/test/test_choicepoint.py (original) +++ pypy/dist/pypy/module/_stackless/test/test_choicepoint.py Tue Sep 25 21:27:07 2007 @@ -1,5 +1,5 @@ -from pypy.module._stackless.interp_coroutine import AbstractThunk -from pypy.module._stackless.interp_clonable import InterpClonableCoroutine as ClonableCoroutine +from pypy.rlib.rcoroutine import AbstractThunk +from pypy.module._stackless.rclonable import InterpClonableCoroutine as ClonableCoroutine class ChoicePointHolder(object): def __init__(self): Modified: pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py (original) +++ pypy/dist/pypy/module/_stackless/test/test_interp_clonable.py Tue Sep 25 21:27:07 2007 @@ -6,8 +6,8 @@ from pypy.translator.c import gc from pypy.rpython.memory.gctransform import stacklessframework from pypy.rpython.memory.test import test_transformed_gc -from pypy.module._stackless.interp_clonable import InterpClonableCoroutine as ClonableCoroutine -from pypy.module._stackless.interp_clonable import AbstractThunk, fork +from pypy.module._stackless.rclonable import InterpClonableCoroutine as ClonableCoroutine +from pypy.module._stackless.rclonable import AbstractThunk, fork class TestClonableCoroutine(test_transformed_gc.GCTest): Modified: pypy/dist/pypy/module/_stackless/test/test_interp_coroutine.py ============================================================================== --- pypy/dist/pypy/module/_stackless/test/test_interp_coroutine.py (original) +++ pypy/dist/pypy/module/_stackless/test/test_interp_coroutine.py Tue Sep 25 21:27:07 2007 @@ -4,7 +4,7 @@ import os from pypy import conftest; conftest.translation_test_so_skip_if_appdirect() -from pypy.module._stackless.interp_coroutine import syncstate, Coroutine, AbstractThunk +from pypy.rlib.rcoroutine import syncstate, Coroutine, AbstractThunk from pypy.translator.c.test.test_stackless import StacklessTest from pypy.translator.c import gc From tismer at codespeak.net Tue Sep 25 22:12:55 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Tue, 25 Sep 2007 22:12:55 +0200 (CEST) Subject: [pypy-svn] r46879 - in pypy/dist/pypy: module/cclp rlib translator/stackless/test Message-ID: <20070925201255.107188150@code0.codespeak.net> Author: tismer Date: Tue Sep 25 22:12:54 2007 New Revision: 46879 Added: pypy/dist/pypy/rlib/rcoroutine.py - copied unchanged from r46862, pypy/dist/pypy/module/_stackless/interp_coroutine.py Modified: pypy/dist/pypy/module/cclp/misc.py pypy/dist/pypy/module/cclp/thunk.py pypy/dist/pypy/translator/stackless/test/test_coroutine_reconstruction.py Log: references to _stackless scattered throughout pypy, hope this is all Modified: pypy/dist/pypy/module/cclp/misc.py ============================================================================== --- pypy/dist/pypy/module/cclp/misc.py (original) +++ pypy/dist/pypy/module/cclp/misc.py Tue Sep 25 22:12:54 2007 @@ -2,7 +2,7 @@ from pypy.rlib.objectmodel import we_are_translated # commonly imported there, used from types, variable, thread -from pypy.module._stackless.coroutine import AppCoroutine +from pypy.module._stackless.interp_coroutine import AppCoroutine import os Modified: pypy/dist/pypy/module/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/module/cclp/thunk.py (original) +++ pypy/dist/pypy/module/cclp/thunk.py Tue Sep 25 22:12:54 2007 @@ -1,5 +1,5 @@ -from pypy.module._stackless.coroutine import _AppThunk, AppCoroutine -from pypy.module._stackless.interp_coroutine import AbstractThunk +from pypy.module._stackless.interp_coroutine import _AppThunk, AppCoroutine +from pypy.rlib.rcoroutine import AbstractThunk from pypy.module.cclp.misc import w, get_current_cspace from pypy.module.cclp.global_state import sched Modified: pypy/dist/pypy/translator/stackless/test/test_coroutine_reconstruction.py ============================================================================== --- pypy/dist/pypy/translator/stackless/test/test_coroutine_reconstruction.py (original) +++ pypy/dist/pypy/translator/stackless/test/test_coroutine_reconstruction.py Tue Sep 25 22:12:54 2007 @@ -1,4 +1,4 @@ -from pypy.module._stackless import interp_coroutine +from pypy.rlib import rcoroutine from pypy.rlib import rstack from pypy.rlib.rstack import resume_state_create from pypy.translator.stackless.test.test_transform import llinterp_stackless_function @@ -8,7 +8,7 @@ class TestCoroutineReconstruction: def setup_meth(self): - interp_coroutine.syncstate.reset() + rcoroutine.syncstate.reset() def test_simple_ish(self): @@ -22,7 +22,7 @@ rstack.resume_point("f_1", coro, n, x) output.append(x) - class T(interp_coroutine.AbstractThunk): + class T(rcoroutine.AbstractThunk): def __init__(self, arg_coro, arg_n, arg_x): self.arg_coro = arg_coro self.arg_n = arg_n @@ -31,17 +31,17 @@ f(self.arg_coro, self.arg_n, self.arg_x) def example(): - main_coro = interp_coroutine.Coroutine.getcurrent() - sub_coro = interp_coroutine.Coroutine() + main_coro = rcoroutine.Coroutine.getcurrent() + sub_coro = rcoroutine.Coroutine() thunk_f = T(main_coro, 5, 1) sub_coro.bind(thunk_f) sub_coro.switch() - new_coro = interp_coroutine.Coroutine() + new_coro = rcoroutine.Coroutine() new_thunk_f = T(main_coro, 5, 1) new_coro.bind(new_thunk_f) - costate = interp_coroutine.Coroutine._get_default_costate() + costate = rcoroutine.Coroutine._get_default_costate() bottom = resume_state_create(None, "yield_current_frame_to_caller_1") _bind_frame = resume_state_create(bottom, "coroutine__bind", costate) f_frame_1 = resume_state_create(_bind_frame, "f_1", main_coro, 5, 1) From arigo at codespeak.net Tue Sep 25 23:07:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 23:07:26 +0200 (CEST) Subject: [pypy-svn] r46882 - in pypy/dist/pypy: module/zlib rlib Message-ID: <20070925210726.1128C814A@code0.codespeak.net> Author: arigo Date: Tue Sep 25 23:07:24 2007 New Revision: 46882 Modified: pypy/dist/pypy/module/zlib/__init__.py pypy/dist/pypy/rlib/rzlib.py Log: Expose some constants on the zlib module. The list is the same as in CPython. Modified: pypy/dist/pypy/module/zlib/__init__.py ============================================================================== --- pypy/dist/pypy/module/zlib/__init__.py (original) +++ pypy/dist/pypy/module/zlib/__init__.py Tue Sep 25 23:07:24 2007 @@ -4,6 +4,7 @@ """ from pypy.interpreter.mixedmodule import MixedModule +from pypy.rlib import rzlib class Module(MixedModule): @@ -14,8 +15,19 @@ 'decompressobj': 'interp_zlib.Decompress', 'compress': 'interp_zlib.compress', 'decompress': 'interp_zlib.decompress', + '__version__': 'space.wrap("1.0")', } appleveldefs = { 'error': 'app_zlib.error', } + + +for _name in """ + MAX_WBITS DEFLATED DEF_MEM_LEVEL + Z_BEST_SPEED Z_BEST_COMPRESSION Z_DEFAULT_COMPRESSION + Z_FILTERED Z_HUFFMAN_ONLY Z_DEFAULT_STRATEGY + Z_FINISH Z_NO_FLUSH Z_SYNC_FLUSH Z_FULL_FLUSH + ZLIB_VERSION + """.split(): + Module.interpleveldefs[_name] = 'space.wrap(%r)' % (getattr(rzlib, _name),) Modified: pypy/dist/pypy/rlib/rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/rzlib.py (original) +++ pypy/dist/pypy/rlib/rzlib.py Tue Sep 25 23:07:24 2007 @@ -7,8 +7,10 @@ constantnames = ''' Z_OK Z_STREAM_ERROR Z_BUF_ERROR Z_MEM_ERROR Z_STREAM_END Z_DEFLATED Z_DEFAULT_STRATEGY Z_DEFAULT_COMPRESSION - Z_NO_FLUSH Z_FINISH Z_SYNC_FLUSH + Z_NO_FLUSH Z_FINISH Z_SYNC_FLUSH Z_FULL_FLUSH MAX_WBITS MAX_MEM_LEVEL + Z_BEST_SPEED Z_BEST_COMPRESSION Z_DEFAULT_COMPRESSION + Z_FILTERED Z_HUFFMAN_ONLY Z_DEFAULT_STRATEGY '''.split() class SimpleCConfig: @@ -43,6 +45,7 @@ # The following parameter is copied from zutil.h, version 0.95, # according to CPython's zlibmodule.c +DEFLATED = Z_DEFLATED if MAX_MEM_LEVEL >= 8: DEF_MEM_LEVEL = 8 else: From arigo at codespeak.net Tue Sep 25 23:08:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 Sep 2007 23:08:34 +0200 (CEST) Subject: [pypy-svn] r46883 - pypy/dist/pypy/config Message-ID: <20070925210834.7863A814B@code0.codespeak.net> Author: arigo Date: Tue Sep 25 23:08:34 2007 New Revision: 46883 Modified: pypy/dist/pypy/config/pypyoption.py Log: Yay! zlib seems to be basically working. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Tue Sep 25 23:08:34 2007 @@ -23,7 +23,7 @@ working_modules = default_modules.copy() working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", - "crypt", "signal", "dyngram", "readline", "termios" + "crypt", "signal", "dyngram", "readline", "termios", "zlib", ] )) From pypy-svn at codespeak.net Wed Sep 26 09:58:53 2007 From: pypy-svn at codespeak.net (Viagra.com Inc) Date: Wed, 26 Sep 2007 09:58:53 +0200 (CEST) Subject: [pypy-svn] September 70% OFF Message-ID: <20030101032525.9385.qmail@static-dup-225.195-146-156.telecom.sk> An HTML attachment was scrubbed... URL: From xoraxax at codespeak.net Wed Sep 26 13:42:03 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 26 Sep 2007 13:42:03 +0200 (CEST) Subject: [pypy-svn] r46896 - in pypy/dist/pypy: bin config lib/pyontology/test objspace/std objspace/std/test translator/goal Message-ID: <20070926114203.F13048108@code0.codespeak.net> Author: xoraxax Date: Wed Sep 26 13:42:02 2007 New Revision: 46896 Modified: pypy/dist/pypy/bin/py.py pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/lib/pyontology/test/test_sparql.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/test/test_index.py pypy/dist/pypy/translator/goal/app_main.py pypy/dist/pypy/translator/goal/targetpypystandalone.py Log: Move the oldstyle translation option to py.py and app_main.py to make it setable at runtime. Modified: pypy/dist/pypy/bin/py.py ============================================================================== --- pypy/dist/pypy/bin/py.py (original) +++ pypy/dist/pypy/bin/py.py Wed Sep 26 13:42:02 2007 @@ -37,6 +37,9 @@ StrOption("runcommand", "program passed in as CMD (terminates option list)", default=None, cmdline="-c"), + BoolOption("oldstyle_classes", + "Use old-style classes by default.", + default=False, cmdline="-k --oldstyle"), ]) pypy_init = gateway.applevel(''' @@ -73,11 +76,9 @@ space = option.make_objspace(config) space._starttime = starttime - #assert 'pypy.tool.udir' not in sys.modules, ( - # "running py.py should not import pypy.tool.udir, which is\n" - # "only for testing or translating purposes.") - # ^^^ _socket and other rctypes-based modules need udir - space.setitem(space.sys.w_dict,space.wrap('executable'),space.wrap(argv[0])) + space.setitem(space.sys.w_dict, space.wrap('executable'), space.wrap(argv[0])) + if interactiveconfig.oldstyle_classes: + space.setitem(space.builtin.w_dict, space.wrap('__metaclass__'), space.w_classobj) # store the command-line arguments into sys.argv go_interactive = interactiveconfig.interactive Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Wed Sep 26 13:42:02 2007 @@ -235,10 +235,6 @@ "special case the 'list[integer]' expressions", default=False), - BoolOption("oldstyle", - "specify whether the default metaclass should be classobj", - default=False, cmdline="--oldstyle"), - BoolOption("logspaceoptypes", "a instrumentation option: before exit, print the types seen by " "certain simpler bytecodes", Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_sparql.py Wed Sep 26 13:42:02 2007 @@ -500,4 +500,4 @@ if check_have_oldstyle(): TestXMLRPC = _TestXMLRPC else: - py.test.skip('Please build PyPy with --oldstyle, needed by xmlrpclib') + py.test.skip('Please run PyPy with --oldstyle, needed by xmlrpclib') Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Wed Sep 26 13:42:02 2007 @@ -48,10 +48,6 @@ PACKAGE_PATH = 'objspace.std' - def setoptions(self, **kwds): - if "oldstyle" in kwds: - self.config.objspace.std.oldstyle = kwds["oldstyle"] - def initialize(self): "NOT_RPYTHON: only for initializing the space." self._typecache = Cache() @@ -253,11 +249,9 @@ """) self.w_dict.__flags__ = old_flags - if self.config.objspace.std.oldstyle: - self.enable_old_style_classes_as_default_metaclass() - # final setup self.setup_builtin_modules() + # Adding transparent proxy call if self.config.objspace.std.withtproxy: w___pypy__ = self.getbuiltinmodule("__pypy__") @@ -268,16 +262,7 @@ self.setattr(w___pypy__, self.wrap('get_tproxy_controller'), self.wrap(app_proxy_controller)) - def enable_old_style_classes_as_default_metaclass(self): - self.setitem(self.builtin.w_dict, self.wrap('__metaclass__'), self.w_classobj) - def enable_new_style_classes_as_default_metaclass(self): - space = self - try: - self.delitem(self.builtin.w_dict, self.wrap('__metaclass__')) - except OperationError, e: - if not e.match(space, space.w_KeyError): - raise def setup_old_style_classes(self): """NOT_RPYTHON""" @@ -292,6 +277,9 @@ self.w_classobj = w_classobj self.w_instance = w_instance + def force_old_style_classes(self): + self.setitem(self.builtin.w_dict, self.wrap('__metaclass__'), self.w_classobj) + def create_builtin_module(self, pyname, publicname): """NOT_RPYTHON helper function which returns the wrapped module and its dict. Modified: pypy/dist/pypy/objspace/std/test/test_index.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_index.py (original) +++ pypy/dist/pypy/objspace/std/test/test_index.py Wed Sep 26 13:42:02 2007 @@ -3,7 +3,9 @@ class AppTest_IndexProtocol: def setup_class(self): - self.space = gettestobjspace(oldstyle=True) + self.space = gettestobjspace() + self.space.force_old_style_classes() + w_oldstyle = self.space.appexec([], """(): class oldstyle: def __index__(self): Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Wed Sep 26 13:42:02 2007 @@ -3,15 +3,16 @@ # See test/test_app_main. """ options: - -i inspect interactively after running script - -O dummy optimization flag for compatibility with C Python - -c CMD program passed in as CMD (terminates option list) - -S do not 'import site' on initialization - -u unbuffered binary stdout and stderr - -h, --help show this help message and exit - -m library module to be run as a script (terminates option list) - --version print the PyPy version - --info print translation information about this PyPy executable + -i inspect interactively after running script + -O dummy optimization flag for compatibility with C Python + -c CMD program passed in as CMD (terminates option list) + -S do not 'import site' on initialization + -u unbuffered binary stdout and stderr + -h, --help show this help message and exit + -m library module to be run as a script (terminates option list) + -k, --oldstyle use old-style classes instead of newstyle classes everywhere + --version print the PyPy version + --info print translation information about this PyPy executable """ import sys, os @@ -179,6 +180,7 @@ i = 0 run_module = False run_stdin = False + oldstyle_classes = False while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -216,6 +218,8 @@ return 2 run_module = True break + elif arg in ('-k', '--oldstyle'): + oldstyle_classes = True elif arg == '--': i += 1 break # terminates option list @@ -235,6 +239,10 @@ mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule + if oldstyle_classes: + import __builtin__ + __builtin__.__metaclass__ = __builtin__._classobj + if import_site: try: import site Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Wed Sep 26 13:42:02 2007 @@ -170,10 +170,6 @@ def get_entry_point(self, config): space = make_objspace(config) - if not config.objspace.std.oldstyle: - # disable translation of the whole of classobjinterp.py - StdObjSpace.setup_old_style_classes = lambda self: None - # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') w_dict = space.newdict() From tismer at codespeak.net Wed Sep 26 14:44:52 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 26 Sep 2007 14:44:52 +0200 (CEST) Subject: [pypy-svn] r46897 - pypy/dist/pypy/module/cclp Message-ID: <20070926124452.370A68169@code0.codespeak.net> Author: tismer Date: Wed Sep 26 14:44:50 2007 New Revision: 46897 Modified: pypy/dist/pypy/module/cclp/cspace.py pypy/dist/pypy/module/cclp/types.py Log: a glitch from yesterday's renaming. Actually I think it is bad that logic space imports from a module's folder. these things are not public. Modified: pypy/dist/pypy/module/cclp/cspace.py ============================================================================== --- pypy/dist/pypy/module/cclp/cspace.py (original) +++ pypy/dist/pypy/module/cclp/cspace.py Wed Sep 26 14:44:50 2007 @@ -17,7 +17,7 @@ from pypy.module._cslib import fd from pypy.rlib.cslib import rdistributor as rd -from pypy.module._stackless.coroutine import AppCoroutine +from pypy.module._stackless.interp_coroutine import AppCoroutine import pypy.rlib.rgc as rgc Modified: pypy/dist/pypy/module/cclp/types.py ============================================================================== --- pypy/dist/pypy/module/cclp/types.py (original) +++ pypy/dist/pypy/module/cclp/types.py Wed Sep 26 14:44:50 2007 @@ -1,6 +1,6 @@ from pypy.interpreter import baseobjspace, gateway, typedef from pypy.interpreter.error import OperationError -from pypy.module._stackless.clonable import AppClonableCoroutine +from pypy.module._stackless.interp_clonable import AppClonableCoroutine from pypy.module.cclp.misc import w, AppCoroutine, get_current_cspace from pypy.module.cclp.global_state import sched From antocuni at codespeak.net Wed Sep 26 16:20:15 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 26 Sep 2007 16:20:15 +0200 (CEST) Subject: [pypy-svn] r46900 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20070926142015.363F4817C@code0.codespeak.net> Author: antocuni Date: Wed Sep 26 16:20:13 2007 New Revision: 46900 Modified: pypy/dist/pypy/translator/backendopt/graphanalyze.py pypy/dist/pypy/translator/backendopt/test/test_canraise.py Log: don't crash when trying to analyze recursive methods Modified: pypy/dist/pypy/translator/backendopt/graphanalyze.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/graphanalyze.py (original) +++ pypy/dist/pypy/translator/backendopt/graphanalyze.py Wed Sep 26 16:20:13 2007 @@ -48,7 +48,7 @@ graph = getattr(meth, 'graph', None) if graph is None: return self.analyze_external_method(op, TYPE, meth) - return self.analyze_oosend(TYPE, name, seen=None) + return self.analyze_oosend(TYPE, name, seen) if self.operation_is_true(op): return True Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_canraise.py Wed Sep 26 16:20:13 2007 @@ -111,6 +111,26 @@ assert not ra.can_raise(op_call_f) assert ra.can_raise(op_call_m) + def test_method_recursive(self): + class A: + def m(self, x): + if x > 0: + return self.m(x-1) + else: + return 42 + def m(a): + return a.m(2) + def h(): + obj = A() + m(obj) + t, ra = self.translate(h, []) + hgraph = graphof(t, h) + # fiiiish :-( + block = hgraph.startblock + op_call_m = block.operations[-1] + assert op_call_m.opname == "direct_call" + assert not ra.can_raise(op_call_m) + def test_instantiate(self): # instantiate is interesting, because it leads to one of the few cases of # an indirect call without a list of graphs From arigo at codespeak.net Wed Sep 26 17:21:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 Sep 2007 17:21:34 +0200 (CEST) Subject: [pypy-svn] r46902 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070926152134.4AEC2816C@code0.codespeak.net> Author: arigo Date: Wed Sep 26 17:21:33 2007 New Revision: 46902 Added: pypy/dist/pypy/module/struct/ (props changed) pypy/dist/pypy/module/struct/__init__.py (contents, props changed) pypy/dist/pypy/module/struct/app_struct.py (contents, props changed) pypy/dist/pypy/module/struct/error.py (contents, props changed) pypy/dist/pypy/module/struct/formatiterator.py (contents, props changed) pypy/dist/pypy/module/struct/interp_struct.py (contents, props changed) pypy/dist/pypy/module/struct/nativefmttable.py (contents, props changed) pypy/dist/pypy/module/struct/standardfmttable.py (contents, props changed) pypy/dist/pypy/module/struct/test/ (props changed) pypy/dist/pypy/module/struct/test/test_struct.py (contents, props changed) Log: Starting an interp-level struct format. Probably trying to pull too many RPython tricks out of my hat in order to get efficient code after translation. Added: pypy/dist/pypy/module/struct/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/__init__.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,54 @@ + +""" +Mixed-module definition for the struct module. +Note that there is also a pure Python implementation in pypy/lib/struct.py; +the present mixed-module version of struct takes precedence if it is enabled. +""" + +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + """\ +Functions to convert between Python values and C structs. +Python strings are used to hold the data representing the C struct +and also as format strings to describe the layout of data in the C struct. + +The optional first format char indicates byte order, size and alignment: + @: native order, size & alignment (default) + =: native order, std. size & alignment + <: little-endian, std. size & alignment + >: big-endian, std. size & alignment + !: same as > + +The remaining chars indicate types of args and must match exactly; +these can be preceded by a decimal repeat count: + x: pad byte (no data); + c:char; + b:signed byte; + B:unsigned byte; + h:short; + H:unsigned short; + i:int; + I:unsigned int; + l:long; + L:unsigned long; + q:long long; + Q:unsigned long long + f:float; + d:double. +Special cases (preceding decimal count indicates length): + s:string (array of char); p: pascal string (with count byte). +Special case (only available in native format): + P:an integer type that is wide enough to hold a pointer. +Whitespace between formats is ignored. + +The variable struct.error is an exception raised on errors.""" + + interpleveldefs = { + 'calcsize': 'interp_struct.calcsize', + } + + appleveldefs = { + 'error': 'app_struct.error', + } Added: pypy/dist/pypy/module/struct/app_struct.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/app_struct.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,10 @@ + +""" +Application-level definitions for the zlib module. + +NOT_RPYTHON +""" + +class error(Exception): + """Exception raised on various occasions; argument is a string + describing what is wrong.""" Added: pypy/dist/pypy/module/struct/error.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/error.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,9 @@ + +class StructError(Exception): + "Interp-level error that gets mapped to an app-level struct.error." + + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg Added: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/formatiterator.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,94 @@ + +from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import ovfcheck + +from pypy.module.struct.error import StructError +from pypy.module.struct.standardfmttable import standard_fmttable +from pypy.module.struct.nativefmttable import native_is_bigendian + + +class FormatIterator(object): + """ + An iterator-like object that follows format strings step by step. + It provides input to the packer/unpacker and accumulates their output. + The subclasses are specialized for either packing, unpacking, or + just computing the size. + """ + _mixin_ = True + + def __init__(self, fmt): + # decode the byte order, size and alignment based on the 1st char + native = True + bigendian = native_is_bigendian + index = 0 + if len(fmt) > 0: + c = fmt[0] + index = 1 + if c == '@': + pass + elif c == '=': + native = False + elif c == '<': + native = False + bigendian = False + elif c == '>' or c == '!': + native = False + bigendian = True + else: + index = 0 + self.native = native + self.bigendian = bigendian + + # immediately interpret the format string, + # calling self.operate() for each format unit + while index < len(fmt): + c = fmt[index] + index += 1 + if c.isspace(): + continue + if c.isdigit(): + repetitions = ord(c) - ord('0') + while True: + if index == len(fmt): + raise StructError("incomplete struct format") + c = fmt[index] + index += 1 + if not c.isdigit(): + break + repetitions = ovfcheck(repetitions * 10) + repetitions = ovfcheck(repetitions + (ord(c) - ord('0'))) + # XXX catch OverflowError somewhere + else: + repetitions = 1 + + for fmtop in unroll_fmtops: + if c == fmtop.fmtchar: + self.operate(fmtop, repetitions) + break + else: + raise StructError("bad char in struct format") + + +class CalcSizeFormatIterator(FormatIterator): + totalsize = 0 + + def operate(self, fmtop, repetitions): + if fmtop.size == 1: + size = repetitions # skip the overflow-checked multiplication by 1 + else: + size = ovfcheck(fmtop.size * repetitions) + self.totalsize = ovfcheck(self.totalsize + size) + operate._annspecialcase_ = 'specialize:argvalue(1)' + + +class FmtOp(object): + def __init__(self, fmtchar, attrs): + self.fmtchar = fmtchar + self.__dict__.update(attrs) + def _freeze_(self): + return True + +_items = standard_fmttable.items() +_items.sort() +unroll_fmtops = unrolling_iterable([FmtOp(_key, _attrs) + for _key, _attrs in _items]) Added: pypy/dist/pypy/module/struct/interp_struct.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/interp_struct.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,8 @@ +from pypy.interpreter.gateway import ObjSpace +from pypy.module.struct.formatiterator import CalcSizeFormatIterator + + +def calcsize(space, format): + fmtiter = CalcSizeFormatIterator(format) + return space.wrap(fmtiter.totalsize) +calcsize.unwrap_spec = [ObjSpace, str] Added: pypy/dist/pypy/module/struct/nativefmttable.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/nativefmttable.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,3 @@ +import struct + +native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1) Added: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/standardfmttable.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,125 @@ +""" +The format table for standard sizes and alignments. +""" + +# Note: we follow Python 2.5 in being strict about the ranges of accepted +# values when packing. + +import struct +from pypy.module.struct.error import StructError +from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong + +# ____________________________________________________________ + +def pack_pad(fmtiter): + fmtiter.result.append('\x00') + +def pack_char(fmtiter): + xxx + +def pack_string(fmtiter): + xxx + +def pack_pascal(fmtiter): + xxx + +def pack_float(fmtiter): + xxx + +# ____________________________________________________________ + +native_int_size = struct.calcsize("l") + +def make_int_packer(size, signed): + if signed: + min = -(2 ** (size-1)) + max = (2 ** (size-1)) - 1 + if size <= native_int_size: + accept_method = 'accept_int_arg' + else: + accept_method = 'accept_longlong_arg' + min = r_longlong(min) + max = r_longlong(max) + else: + min = 0 + max = (2 ** size) - 1 + if size < native_int_size: + accept_method = 'accept_int_arg' + elif size == native_int_size: + accept_method = 'accept_uint_arg' + min = r_uint(min) + max = r_uint(max) + else: + accept_method = 'accept_ulonglong_arg' + min = r_ulonglong(min) + max = r_ulonglong(max) + if size > 1: + plural = "s" + else: + plural = "" + errormsg = "argument out of range for %d-byte%s integer format" % (size, + plural) + unroll_range_size = unrolling_iterable(range(size)) + + def pack_int(fmtiter): + method = getattr(fmtiter, accept_method) + value = method() + if value < min or value > max: + raise StructError(errormsg) + if fmtiter.bigendian: + for i in unroll_range_size: + x = (value >> (8*i)) & 0xff + fmtiter.result.append(chr(x)) + else: + for i in unroll_range_size: + fmtiter.result.append(chr(value & 0xff)) + value >>= 8 + + return pack_int + +# ____________________________________________________________ + +def unpack_pad(fmtiter): + xxx + +def unpack_char(fmtiter): + xxx + +def unpack_string(fmtiter): + xxx + +def unpack_pascal(fmtiter): + xxx + +def unpack_float(fmtiter): + xxx + +# ____________________________________________________________ + +def make_int_unpacker(size, signed): + return lambda fmtiter: xxx + +# ____________________________________________________________ + +standard_fmttable = { + 'x':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_pad, 'unpack' : unpack_pad}, + 'c':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_char, 'unpack' : unpack_char}, + 's':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_string, 'unpack' : unpack_string}, + 'p':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_pascal, 'unpack' : unpack_pascal}, + 'f':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float}, + 'd':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float}, + } + +for c, size in [('b', 1), ('h', 2), ('i', 4), ('q', 8)]: # 'l' see below + standard_fmttable[c] = {'size': size, + 'alignment': 0, + 'pack': make_int_packer(size, True), + 'unpack': make_int_unpacker(size, True)} + standard_fmttable[c.upper()] = {'size': size, + 'alignment': 0, + 'pack': make_int_packer(size, False), + 'unpack': make_int_unpacker(size, False)} + +standard_fmttable['l'] = standard_fmttable['i'] +standard_fmttable['L'] = standard_fmttable['I'] Added: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/test/test_struct.py Wed Sep 26 17:21:33 2007 @@ -0,0 +1,73 @@ +""" +Tests for the struct module implemented at interp-level in pypy/module/struct. +""" + +import py +from pypy.conftest import gettestobjspace + + +class AppTestStruct(object): + + def setup_class(cls): + """ + Create a space with the struct module and import it for use by the + tests. + """ + cls.space = gettestobjspace(usemodules=['struct']) + cls.w_struct = cls.space.call_function( + cls.space.builtin.get('__import__'), + cls.space.wrap('struct')) + + + def test_error(self): + """ + struct.error should be an exception class. + """ + assert issubclass(self.struct.error, Exception) + + + def test_calcsize_standard(self): + """ + Check the standard size of the various format characters. + """ + calcsize = self.struct.calcsize + assert calcsize('=') == 0 + assert calcsize('c') == 1 + assert calcsize('!b') == 1 + assert calcsize('=B') == 1 + assert calcsize('H') == 2 + assert calcsize('!i') == 4 + assert calcsize('=I') == 4 + assert calcsize('L') == 4 + assert calcsize('!q') == 8 + assert calcsize('=Q') == 8 + assert calcsize('d') == 8 + assert calcsize('!13s') == 13 + assert calcsize('=500p') == 500 + # test with some repetitions and multiple format characters + assert calcsize('=bQ3i') == 1 + 8 + 3*4 + + + def test_calcsize_native(self): + """ + Check that the size of the various format characters is reasonable. + """ + skip("in-progress") + calcsize = self.struct.calcsize + assert calcsize('') == 0 + assert calcsize('x') == 1 + assert calcsize('c') == 1 + assert calcsize('b') == 1 + assert calcsize('B') == 1 + assert (2 <= calcsize('h') == calcsize('H') + < calcsize('i') == calcsize('I') + <= calcsize('l') == calcsize('L') + <= calcsize('q') == calcsize('Q')) + assert 4 <= calcsize('f') <= 8 <= calcsize('d') + assert calcsize('13s') == 13 + assert calcsize('500p') == 500 + assert 4 <= calcsize('P') <= 8 From arigo at codespeak.net Wed Sep 26 17:30:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 Sep 2007 17:30:04 +0200 (CEST) Subject: [pypy-svn] r46908 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070926153004.9784F80A9@code0.codespeak.net> Author: arigo Date: Wed Sep 26 17:30:04 2007 New Revision: 46908 Modified: pypy/dist/pypy/module/struct/__init__.py pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/interp_struct.py pypy/dist/pypy/module/struct/standardfmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: First test about struct.pack(). Modified: pypy/dist/pypy/module/struct/__init__.py ============================================================================== --- pypy/dist/pypy/module/struct/__init__.py (original) +++ pypy/dist/pypy/module/struct/__init__.py Wed Sep 26 17:30:04 2007 @@ -47,6 +47,7 @@ interpleveldefs = { 'calcsize': 'interp_struct.calcsize', + 'pack': 'interp_struct.pack', } appleveldefs = { Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Wed Sep 26 17:30:04 2007 @@ -16,7 +16,7 @@ """ _mixin_ = True - def __init__(self, fmt): + def interpret(self, fmt): # decode the byte order, size and alignment based on the 1st char native = True bigendian = native_is_bigendian @@ -38,8 +38,7 @@ index = 0 self.native = native self.bigendian = bigendian - - # immediately interpret the format string, + # interpret the format string, # calling self.operate() for each format unit while index < len(fmt): c = fmt[index] @@ -81,6 +80,23 @@ operate._annspecialcase_ = 'specialize:argvalue(1)' +class PackFormatIterator(FormatIterator): + + def __init__(self, space, args_w): + self.space = space + self.args_iterator = iter(args_w) + self.result = [] # list of characters + + def operate(self, fmtop, repetitions): + # XXX 's', 'p' + for i in range(repetitions): + fmtop.pack(self) + + def accept_int_arg(self): + w_obj = self.args_iterator.next() # XXX catch StopIteration + return self.space.int_w(w_obj) + + class FmtOp(object): def __init__(self, fmtchar, attrs): self.fmtchar = fmtchar Modified: pypy/dist/pypy/module/struct/interp_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/interp_struct.py (original) +++ pypy/dist/pypy/module/struct/interp_struct.py Wed Sep 26 17:30:04 2007 @@ -1,8 +1,18 @@ from pypy.interpreter.gateway import ObjSpace from pypy.module.struct.formatiterator import CalcSizeFormatIterator +from pypy.module.struct.formatiterator import PackFormatIterator def calcsize(space, format): - fmtiter = CalcSizeFormatIterator(format) + fmtiter = CalcSizeFormatIterator() + fmtiter.interpret(format) return space.wrap(fmtiter.totalsize) calcsize.unwrap_spec = [ObjSpace, str] + + +def pack(space, format, args_w): + fmtiter = PackFormatIterator(space, args_w) + fmtiter.interpret(format) + result = ''.join(fmtiter.result) + return space.wrap(result) +pack.unwrap_spec = [ObjSpace, str, 'args_w'] Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Wed Sep 26 17:30:04 2007 @@ -33,8 +33,8 @@ def make_int_packer(size, signed): if signed: - min = -(2 ** (size-1)) - max = (2 ** (size-1)) - 1 + min = -(2 ** (8*size-1)) + max = (2 ** (8*size-1)) - 1 if size <= native_int_size: accept_method = 'accept_int_arg' else: @@ -43,7 +43,7 @@ max = r_longlong(max) else: min = 0 - max = (2 ** size) - 1 + max = (2 ** (8*size)) - 1 if size < native_int_size: accept_method = 'accept_int_arg' elif size == native_int_size: Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Wed Sep 26 17:30:04 2007 @@ -52,6 +52,14 @@ assert calcsize('=bQ3i') == 1 + 8 + 3*4 + def test_pack_standard_little(self): + """ + Check packing with the '<' format specifier. + """ + pack = self.struct.pack + assert pack(" Author: arigo Date: Wed Sep 26 18:05:59 2007 New Revision: 46909 Modified: pypy/dist/pypy/module/struct/__init__.py pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/interp_struct.py pypy/dist/pypy/module/struct/standardfmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: A first test about unpacking. Modified: pypy/dist/pypy/module/struct/__init__.py ============================================================================== --- pypy/dist/pypy/module/struct/__init__.py (original) +++ pypy/dist/pypy/module/struct/__init__.py Wed Sep 26 18:05:59 2007 @@ -48,6 +48,7 @@ interpleveldefs = { 'calcsize': 'interp_struct.calcsize', 'pack': 'interp_struct.pack', + 'unpack': 'interp_struct.unpack', } appleveldefs = { Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Wed Sep 26 18:05:59 2007 @@ -91,12 +91,40 @@ # XXX 's', 'p' for i in range(repetitions): fmtop.pack(self) + operate._annspecialcase_ = 'specialize:argvalue(1)' def accept_int_arg(self): w_obj = self.args_iterator.next() # XXX catch StopIteration return self.space.int_w(w_obj) +class UnpackFormatIterator(FormatIterator): + + def __init__(self, space, input): + self.space = space + self.input = input + self.inputpos = 0 + self.result_w = [] # list of wrapped objects + + def operate(self, fmtop, repetitions): + # XXX 's', 'p' + for i in range(repetitions): + fmtop.unpack(self) + operate._annspecialcase_ = 'specialize:argvalue(1)' + + def read(self, count): + end = self.inputpos + count + if end > len(self.input): + raise StructError("unpack str size is too short for the format") + s = self.input[self.inputpos : end] + self.inputpos = end + return s + + def append(self, value): + self.result_w.append(self.space.wrap(value)) + append._annspecialcase_ = 'specialize:argtype(1)' + + class FmtOp(object): def __init__(self, fmtchar, attrs): self.fmtchar = fmtchar Modified: pypy/dist/pypy/module/struct/interp_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/interp_struct.py (original) +++ pypy/dist/pypy/module/struct/interp_struct.py Wed Sep 26 18:05:59 2007 @@ -1,6 +1,7 @@ from pypy.interpreter.gateway import ObjSpace from pypy.module.struct.formatiterator import CalcSizeFormatIterator from pypy.module.struct.formatiterator import PackFormatIterator +from pypy.module.struct.formatiterator import UnpackFormatIterator def calcsize(space, format): @@ -13,6 +14,15 @@ def pack(space, format, args_w): fmtiter = PackFormatIterator(space, args_w) fmtiter.interpret(format) + # XXX check that all arguments have been consumed result = ''.join(fmtiter.result) return space.wrap(result) pack.unwrap_spec = [ObjSpace, str, 'args_w'] + + +def unpack(space, format, input): + fmtiter = UnpackFormatIterator(space, input) + fmtiter.interpret(format) + # XXX check that the input string has been fully consumed + return space.newtuple(fmtiter.result_w) +unpack.unwrap_spec = [ObjSpace, str, str] Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Wed Sep 26 18:05:59 2007 @@ -60,7 +60,7 @@ plural = "" errormsg = "argument out of range for %d-byte%s integer format" % (size, plural) - unroll_range_size = unrolling_iterable(range(size)) + unroll_revrange_size = unrolling_iterable(range(size-1, -1, -1)) def pack_int(fmtiter): method = getattr(fmtiter, accept_method) @@ -68,11 +68,11 @@ if value < min or value > max: raise StructError(errormsg) if fmtiter.bigendian: - for i in unroll_range_size: + for i in unroll_revrange_size: x = (value >> (8*i)) & 0xff fmtiter.result.append(chr(x)) else: - for i in unroll_range_size: + for i in unroll_revrange_size: fmtiter.result.append(chr(value & 0xff)) value >>= 8 @@ -98,7 +98,36 @@ # ____________________________________________________________ def make_int_unpacker(size, signed): - return lambda fmtiter: xxx + if signed: + if size <= native_int_size: + inttype = int + else: + inttype = r_longlong + else: + if size < native_int_size: + inttype = int + elif size == native_int_size: + inttype = r_uint + else: + inttype = r_ulonglong + unroll_range_size = unrolling_iterable(range(size)) + + def unpack_int(fmtiter): + intvalue = inttype(0) + s = fmtiter.read(size) + idx = 0 + if fmtiter.bigendian: + for i in unroll_range_size: + intvalue <<= 8 + intvalue |= ord(s[idx]) + idx += 1 + else: + for i in unroll_range_size: + intvalue |= inttype(ord(s[idx])) << (8*i) + idx += 1 + fmtiter.append(intvalue) + + return unpack_int # ____________________________________________________________ Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Wed Sep 26 18:05:59 2007 @@ -60,6 +60,14 @@ assert pack(" An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Wed Sep 26 21:47:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 26 Sep 2007 21:47:14 +0200 (CEST) Subject: [pypy-svn] r46911 - in pypy/dist/pypy/translator: pyrex tool Message-ID: <20070926194714.CBF1A8170@code0.codespeak.net> Author: cfbolz Date: Wed Sep 26 21:47:13 2007 New Revision: 46911 Removed: pypy/dist/pypy/translator/pyrex/ Modified: pypy/dist/pypy/translator/tool/cbuild.py Log: delete the pyrex backend and some code that imports it. Not completely sure I got every reference. Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Wed Sep 26 21:47:13 2007 @@ -10,20 +10,6 @@ debug = 0 -def make_module_from_pyxstring(name, dirpath, string): - dirpath = py.path.local(dirpath) - pyxfile = dirpath.join('%s.pyx' % name) - i = 0 - while pyxfile.check(): - pyxfile = pyxfile.new(basename='%s%d.pyx' % (name, i)) - i+=1 - pyxfile.write(string) - if debug: print "made pyxfile", pyxfile - cfile = make_c_from_pyxfile(pyxfile) - module = make_module_from_c(cfile) - #print "made module", module - return module - def compiler_command(): # e.g. for tcc, you might set this to # "tcc -shared -o %s.so %s.c" @@ -185,89 +171,6 @@ file.close() return mod -def make_c_from_pyxfile(pyxfile): - from pypy.translator.pyrex import genpyrex - pyrexdir = os.path.dirname(genpyrex.__file__) - if pyrexdir not in sys.path: - sys.path.insert(0, pyrexdir) - from Pyrex.Compiler.Main import CompilationOptions, Context, PyrexError - try: - options = CompilationOptions(show_version = 0, - use_listing_file = 0, - c_only = 1, - output_file = None) - context = Context(options.include_path) - result = context.compile(str(pyxfile), options) - if result.num_errors > 0: - raise ValueError, "failure %s" % result - except PyrexError, e: - print >>sys.stderr, e - cfile = pyxfile.new(ext='.c') - return cfile - -def build_cfunc(func, simplify=1, dot=1, inputargtypes=None): - """ return a pyrex-generated cfunction from the given func. - - simplify is true -> perform simplifications on the flowgraph. - dot is true -> generate a dot-configuration file and postscript. - inputargtypes is a list (allowed to be empty) -> - then annotation will be performed before generating - dot/pyrex/c code. - - """ - try: func = func.im_func - except AttributeError: pass - - # build the flow graph - from pypy.objspace.flow import Space - from pypy.tool.udir import udir - space = Space() - name = func.func_name - funcgraph = space.build_flow(func) - - if not inputargtypes: - source = inspect.getsource(func) - base = udir.join(name).new(ext='.py').write(source) - - if dot: - from pypy.translator.tool.make_dot import FlowGraphDotGen - dotgen = FlowGraphDotGen(name) - dotgen.emit_subgraph(name, funcgraph) - - # apply transformations - if simplify: - from pypy.translator.simplify import simplify_graph - simplify_graph(funcgraph) - name += '_s' - - # get the pyrex generator - from pypy.translator.pyrex.genpyrex import GenPyrex - genpyrex = GenPyrex(funcgraph) - - # generate pyrex (without type inference) - - # apply type inference - if inputargtypes is not None: - genpyrex.annotate(inputargtypes) - name += '_t' - #a = Annotator(self.functiongraph) - #a.build_types(input_arg_types) - #a.simplify() - - pyxstring = genpyrex.emitcode() - #funcgraph.source = inspect.getsource(func) - else: - pyxstring = genpyrex.emitcode() - - pyxheader = genpyrex.globaldeclarations() - mod = make_module_from_pyxstring(name, udir, pyxheader + '\n' + pyxstring) - - if dot: - if name != func.func_name: # if some transformations have been done - dotgen.emit_subgraph(name, funcgraph) - dotgen.generate() - return getattr(mod, func.func_name) - def log_spawned_cmd(spawn): def spawn_and_log(cmd, *args, **kwds): From xoraxax at codespeak.net Wed Sep 26 23:21:42 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 26 Sep 2007 23:21:42 +0200 (CEST) Subject: [pypy-svn] r46914 - in pypy/dist/pypy/tool: . test Message-ID: <20070926212142.881E4815E@code0.codespeak.net> Author: xoraxax Date: Wed Sep 26 23:21:41 2007 New Revision: 46914 Modified: pypy/dist/pypy/tool/import_graph.py pypy/dist/pypy/tool/test/test_tab.py Log: Removed two remaining pyrex references. Modified: pypy/dist/pypy/tool/import_graph.py ============================================================================== --- pypy/dist/pypy/tool/import_graph.py (original) +++ pypy/dist/pypy/tool/import_graph.py Wed Sep 26 23:21:41 2007 @@ -10,7 +10,7 @@ def include_file(path): if ("test" in str(path) or "tool" in str(path) or - "documentation" in str(path) or "pyrex" in str(path) or + "documentation" in str(path) or "_cache" in str(path)): return False if path.basename in exclude_files: Modified: pypy/dist/pypy/tool/test/test_tab.py ============================================================================== --- pypy/dist/pypy/tool/test/test_tab.py (original) +++ pypy/dist/pypy/tool/test/test_tab.py Wed Sep 26 23:21:41 2007 @@ -6,7 +6,7 @@ import os ROOT = autopath.pypydir -EXCLUDE = {'/translator/pyrex/Pyrex': True} +EXCLUDE = {} def test_no_tabs(): From arigo at codespeak.net Thu Sep 27 09:51:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 09:51:31 +0200 (CEST) Subject: [pypy-svn] r46919 - pypy/dist/pypy/module/zlib Message-ID: <20070927075131.88BC58150@code0.codespeak.net> Author: arigo Date: Thu Sep 27 09:51:31 2007 New Revision: 46919 Modified: pypy/dist/pypy/module/zlib/__init__.py Log: Module docstring. Modified: pypy/dist/pypy/module/zlib/__init__.py ============================================================================== --- pypy/dist/pypy/module/zlib/__init__.py (original) +++ pypy/dist/pypy/module/zlib/__init__.py Thu Sep 27 09:51:31 2007 @@ -8,6 +8,21 @@ class Module(MixedModule): + """\ +The functions in this module allow compression and decompression using the +zlib library, which is based on GNU zip. + +adler32(string[, start]) -- Compute an Adler-32 checksum. +compress(string[, level]) -- Compress string, with compression level in 1-9. +compressobj([level]) -- Return a compressor object. +crc32(string[, start]) -- Compute a CRC-32 checksum. +decompress(string,[wbits],[bufsize]) -- Decompresses a compressed string. +decompressobj([wbits]) -- Return a decompressor object. + +'wbits' is window buffer size. +Compressor objects support compress() and flush() methods; decompressor +objects support decompress() and flush().""" + interpleveldefs = { 'crc32': 'interp_zlib.crc32', 'adler32': 'interp_zlib.adler32', From arigo at codespeak.net Thu Sep 27 09:51:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 09:51:46 +0200 (CEST) Subject: [pypy-svn] r46920 - pypy/dist/pypy/module/zlib/test Message-ID: <20070927075146.A927E816B@code0.codespeak.net> Author: arigo Date: Thu Sep 27 09:51:46 2007 New Revision: 46920 Modified: pypy/dist/pypy/module/zlib/test/test_zlib.py Log: Allow these tests to run on "py.test -A". Modified: pypy/dist/pypy/module/zlib/test/test_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/test/test_zlib.py (original) +++ pypy/dist/pypy/module/zlib/test/test_zlib.py Thu Sep 27 09:51:46 2007 @@ -16,9 +16,10 @@ against. """ cls.space = gettestobjspace(usemodules=['zlib']) - cls.w_zlib = cls.space.call_function( - cls.space.builtin.get('__import__'), - cls.space.wrap('zlib')) + cls.w_zlib = cls.space.appexec([], """(): + import zlib + return zlib + """) expanded = 'some bytes which will be compressed' cls.w_expanded = cls.space.wrap(expanded) cls.w_compressed = cls.space.wrap(zlib.compress(expanded)) From arigo at codespeak.net Thu Sep 27 09:52:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 09:52:08 +0200 (CEST) Subject: [pypy-svn] r46921 - in pypy/dist/pypy/module/zlib: . test Message-ID: <20070927075208.3188A815D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 09:52:07 2007 New Revision: 46921 Modified: pypy/dist/pypy/module/zlib/ (props changed) pypy/dist/pypy/module/zlib/__init__.py (props changed) pypy/dist/pypy/module/zlib/app_zlib.py (props changed) pypy/dist/pypy/module/zlib/interp_zlib.py (props changed) pypy/dist/pypy/module/zlib/test/ (props changed) pypy/dist/pypy/module/zlib/test/test_zlib.py (props changed) Log: fixeol From arigo at codespeak.net Thu Sep 27 09:53:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 09:53:15 +0200 (CEST) Subject: [pypy-svn] r46922 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070927075315.E1D6F815D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 09:53:15 2007 New Revision: 46922 Modified: pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/nativefmttable.py pypy/dist/pypy/module/struct/standardfmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: * native sizes and alignment, computed with rffi_platform * more formatters Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Thu Sep 27 09:53:15 2007 @@ -4,6 +4,7 @@ from pypy.module.struct.error import StructError from pypy.module.struct.standardfmttable import standard_fmttable +from pypy.module.struct.nativefmttable import native_fmttable from pypy.module.struct.nativefmttable import native_is_bigendian @@ -18,8 +19,8 @@ def interpret(self, fmt): # decode the byte order, size and alignment based on the 1st char - native = True - bigendian = native_is_bigendian + table = unroll_native_fmtdescs + self.bigendian = native_is_bigendian index = 0 if len(fmt) > 0: c = fmt[0] @@ -27,17 +28,16 @@ if c == '@': pass elif c == '=': - native = False + table = unroll_standard_fmtdescs elif c == '<': - native = False - bigendian = False + table = unroll_standard_fmtdescs + self.bigendian = False elif c == '>' or c == '!': - native = False - bigendian = True + table = unroll_standard_fmtdescs + self.bigendian = True else: index = 0 - self.native = native - self.bigendian = bigendian + # interpret the format string, # calling self.operate() for each format unit while index < len(fmt): @@ -60,9 +60,11 @@ else: repetitions = 1 - for fmtop in unroll_fmtops: - if c == fmtop.fmtchar: - self.operate(fmtop, repetitions) + for fmtdesc in table: + if c == fmtdesc.fmtchar: + if fmtdesc.alignment > 1: + self.align(fmtdesc.mask) + self.operate(fmtdesc, repetitions) break else: raise StructError("bad char in struct format") @@ -71,14 +73,18 @@ class CalcSizeFormatIterator(FormatIterator): totalsize = 0 - def operate(self, fmtop, repetitions): - if fmtop.size == 1: + def operate(self, fmtdesc, repetitions): + if fmtdesc.size == 1: size = repetitions # skip the overflow-checked multiplication by 1 else: - size = ovfcheck(fmtop.size * repetitions) + size = ovfcheck(fmtdesc.size * repetitions) self.totalsize = ovfcheck(self.totalsize + size) operate._annspecialcase_ = 'specialize:argvalue(1)' + def align(self, mask): + pad = (-self.totalsize) & mask + self.totalsize = ovfcheck(self.totalsize + pad) + class PackFormatIterator(FormatIterator): @@ -87,16 +93,27 @@ self.args_iterator = iter(args_w) self.result = [] # list of characters - def operate(self, fmtop, repetitions): - # XXX 's', 'p' - for i in range(repetitions): - fmtop.pack(self) + def operate(self, fmtdesc, repetitions): + if fmtdesc.needcount: + fmtdesc.pack(self, repetitions) + else: + for i in range(repetitions): + fmtdesc.pack(self) operate._annspecialcase_ = 'specialize:argvalue(1)' + def align(self, mask): + pad = (-len(self.result)) & mask + for i in range(pad): + self.result.append('\x00') + def accept_int_arg(self): w_obj = self.args_iterator.next() # XXX catch StopIteration return self.space.int_w(w_obj) + def accept_str_arg(self): + w_obj = self.args_iterator.next() # XXX catch StopIteration + return self.space.str_w(w_obj) + class UnpackFormatIterator(FormatIterator): @@ -106,12 +123,17 @@ self.inputpos = 0 self.result_w = [] # list of wrapped objects - def operate(self, fmtop, repetitions): - # XXX 's', 'p' - for i in range(repetitions): - fmtop.unpack(self) + def operate(self, fmtdesc, repetitions): + if fmtdesc.needcount: + fmtdesc.unpack(self, repetitions) + else: + for i in range(repetitions): + fmtdesc.unpack(self) operate._annspecialcase_ = 'specialize:argvalue(1)' + def align(self, mask): + self.inputpos = (self.inputpos + mask) & ~mask + def read(self, count): end = self.inputpos + count if end > len(self.input): @@ -120,19 +142,28 @@ self.inputpos = end return s - def append(self, value): + def appendobj(self, value): self.result_w.append(self.space.wrap(value)) - append._annspecialcase_ = 'specialize:argtype(1)' + appendobj._annspecialcase_ = 'specialize:argtype(1)' -class FmtOp(object): +class FmtDesc(object): def __init__(self, fmtchar, attrs): self.fmtchar = fmtchar + self.alignment = 1 # by default + self.needcount = False # by default self.__dict__.update(attrs) + self.mask = self.alignment - 1 + assert self.alignment & self.mask == 0, ( + "this module assumes that all alignments are powers of two") def _freeze_(self): return True -_items = standard_fmttable.items() -_items.sort() -unroll_fmtops = unrolling_iterable([FmtOp(_key, _attrs) - for _key, _attrs in _items]) +def table2desclist(table): + items = table.items() + items.sort() + lst = [FmtDesc(key, attrs) for key, attrs in items] + return unrolling_iterable(lst) + +unroll_standard_fmtdescs = table2desclist(standard_fmttable) +unroll_native_fmtdescs = table2desclist(native_fmttable) Modified: pypy/dist/pypy/module/struct/nativefmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/nativefmttable.py (original) +++ pypy/dist/pypy/module/struct/nativefmttable.py Thu Sep 27 09:53:15 2007 @@ -1,3 +1,62 @@ import struct +from pypy.module.struct.error import StructError +from pypy.module.struct import standardfmttable as std +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import lltype, rffi native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1) + +native_fmttable = { + 'x': std.standard_fmttable['x'], + 'c': std.standard_fmttable['c'], + 's': std.standard_fmttable['s'], + 'p': std.standard_fmttable['p'], + } + +# ____________________________________________________________ +# +# Use rffi_platform to get the native sizes and alignments from the C compiler + +INSPECT = {'b': 'signed char', + 'h': 'signed short', + 'i': 'signed int', + 'l': 'signed long', + 'q': 'signed long long', + 'B': 'unsigned char', + 'H': 'unsigned short', + 'I': 'unsigned int', + 'L': 'unsigned long', + 'Q': 'unsigned long long', + 'P': 'char *', + 'f': 'float', + 'd': 'double', + } + +class CConfig: + _header_ = "" + +for fmtchar, ctype in INSPECT.items(): + CConfig._header_ += """ + struct about_%s { + char pad; + %s field; + }; + """ % (fmtchar, ctype) + setattr(CConfig, fmtchar, rffi_platform.Struct( + "struct about_%s" % (fmtchar,), + [('field', lltype.FixedSizeArray(rffi.CHAR, 1))])) + +cConfig = rffi_platform.configure(CConfig) + +for fmtchar, ctype in INSPECT.items(): + S = cConfig[fmtchar] + alignment = rffi.offsetof(S, 'c_field') + size = rffi.sizeof(S.c_field) + signed = 'a' <= fmtchar <= 'z' + + native_fmttable[fmtchar] = { + 'size': size, + 'alignment': alignment, + 'pack': std.make_int_packer(size, signed), # XXX 'f', 'd' + 'unpack': std.make_int_unpacker(size, signed), + } Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Thu Sep 27 09:53:15 2007 @@ -12,17 +12,41 @@ # ____________________________________________________________ -def pack_pad(fmtiter): - fmtiter.result.append('\x00') +def pack_pad(fmtiter, count): + for i in range(count): + fmtiter.result.append('\x00') def pack_char(fmtiter): - xxx - -def pack_string(fmtiter): - xxx + string = fmtiter.accept_str_arg() + if len(string) != 1: + raise StructError("expected a string of length 1") + c = string[0] # string->char conversion for the annotator + fmtiter.result.append(c) + +def pack_string(fmtiter, count): + string = fmtiter.accept_str_arg() + if len(string) < count: + fmtiter.result += string + for i in range(len(string), count): + fmtiter.result.append('\x00') + else: + fmtiter.result += string[:count] -def pack_pascal(fmtiter): - xxx +def pack_pascal(fmtiter, count): + string = fmtiter.accept_str_arg() + prefix = len(string) + if prefix >= count: + prefix = count - 1 + if prefix < 0: + raise StructError("bad '0p' in struct format") + if prefix > 255: + prefixchar = '\xff' + else: + prefixchar = chr(prefix) + fmtiter.result.append(prefixchar) + fmtiter.result += string[:prefix] + for i in range(1 + prefix, count): + fmtiter.result.append('\x00') def pack_float(fmtiter): xxx @@ -31,7 +55,11 @@ native_int_size = struct.calcsize("l") -def make_int_packer(size, signed): +def make_int_packer(size, signed, _memo={}): + try: + return _memo[size, signed] + except KeyError: + pass if signed: min = -(2 ** (8*size-1)) max = (2 ** (8*size-1)) - 1 @@ -76,34 +104,47 @@ fmtiter.result.append(chr(value & 0xff)) value >>= 8 + _memo[size, signed] = pack_int return pack_int # ____________________________________________________________ -def unpack_pad(fmtiter): - xxx +def unpack_pad(fmtiter, count): + fmtiter.read(count) def unpack_char(fmtiter): - xxx + fmtiter.appendobj(fmtiter.read(1)) -def unpack_string(fmtiter): - xxx +def unpack_string(fmtiter, count): + fmtiter.appendobj(fmtiter.read(count)) -def unpack_pascal(fmtiter): - xxx +def unpack_pascal(fmtiter, count): + if count == 0: + raise StructError("bad '0p' in struct format") + data = fmtiter.read(count) + end = 1 + ord(data[0]) + if end > count: + end = count + fmtiter.appendobj(data[1:end]) def unpack_float(fmtiter): xxx # ____________________________________________________________ -def make_int_unpacker(size, signed): +def make_int_unpacker(size, signed, _memo={}): + try: + return _memo[size, signed] + except KeyError: + pass if signed: + max = (2 ** (8*size-1)) - 1 if size <= native_int_size: inttype = int else: inttype = r_longlong else: + max = None if size < native_int_size: inttype = int elif size == native_int_size: @@ -125,30 +166,33 @@ for i in unroll_range_size: intvalue |= inttype(ord(s[idx])) << (8*i) idx += 1 - fmtiter.append(intvalue) + if max is not None and intvalue > max: + intvalue -= max + intvalue -= max + intvalue -= 2 + fmtiter.appendobj(intvalue) + _memo[size, signed] = unpack_int return unpack_int # ____________________________________________________________ standard_fmttable = { - 'x':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_pad, 'unpack' : unpack_pad}, - 'c':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_char, 'unpack' : unpack_char}, - 's':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_string, 'unpack' : unpack_string}, - 'p':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_pascal, 'unpack' : unpack_pascal}, - 'f':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float}, - 'd':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float}, + 'x':{ 'size' : 1, 'pack' : pack_pad, 'unpack' : unpack_pad, + 'needcount' : True }, + 'c':{ 'size' : 1, 'pack' : pack_char, 'unpack' : unpack_char}, + 's':{ 'size' : 1, 'pack' : pack_string, 'unpack' : unpack_string, + 'needcount' : True }, + 'p':{ 'size' : 1, 'pack' : pack_pascal, 'unpack' : unpack_pascal, + 'needcount' : True }, + 'f':{ 'size' : 4, 'pack' : pack_float, 'unpack' : unpack_float}, + 'd':{ 'size' : 8, 'pack' : pack_float, 'unpack' : unpack_float}, } -for c, size in [('b', 1), ('h', 2), ('i', 4), ('q', 8)]: # 'l' see below +for c, size in [('b', 1), ('h', 2), ('i', 4), ('l', 4), ('q', 8)]: standard_fmttable[c] = {'size': size, - 'alignment': 0, 'pack': make_int_packer(size, True), 'unpack': make_int_unpacker(size, True)} standard_fmttable[c.upper()] = {'size': size, - 'alignment': 0, 'pack': make_int_packer(size, False), 'unpack': make_int_unpacker(size, False)} - -standard_fmttable['l'] = standard_fmttable['i'] -standard_fmttable['L'] = standard_fmttable['I'] Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Thu Sep 27 09:53:15 2007 @@ -4,6 +4,7 @@ import py from pypy.conftest import gettestobjspace +from pypy.module.struct.nativefmttable import native_is_bigendian class AppTestStruct(object): @@ -14,9 +15,11 @@ tests. """ cls.space = gettestobjspace(usemodules=['struct']) - cls.w_struct = cls.space.call_function( - cls.space.builtin.get('__import__'), - cls.space.wrap('struct')) + cls.w_struct = cls.space.appexec([], """(): + import struct + return struct + """) + cls.w_native_is_bigendian = cls.space.wrap(native_is_bigendian) def test_error(self): @@ -58,6 +61,8 @@ """ pack = self.struct.pack assert pack(" Author: arigo Date: Thu Sep 27 09:55:26 2007 New Revision: 46923 Modified: pypy/dist/pypy/module/struct/test/test_struct.py Log: An extra test. Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Thu Sep 27 09:55:26 2007 @@ -99,6 +99,10 @@ assert calcsize('bi') == calcsize('ii') == 2 * calcsize('i') assert calcsize('bbi') == calcsize('ii') == 2 * calcsize('i') assert calcsize('hi') == calcsize('ii') == 2 * calcsize('i') + # CPython adds no padding at the end, unlike a C compiler + assert calcsize('ib') == calcsize('i') + calcsize('b') + assert calcsize('ibb') == calcsize('i') + 2 * calcsize('b') + assert calcsize('ih') == calcsize('i') + calcsize('h') def test_pack_native(self): From arigo at codespeak.net Thu Sep 27 10:17:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 10:17:39 +0200 (CEST) Subject: [pypy-svn] r46925 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070927081739.47CE3813D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 10:17:38 2007 New Revision: 46925 Modified: pypy/dist/pypy/module/struct/error.py pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/interp_struct.py pypy/dist/pypy/module/struct/nativefmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: Report the error conditions at app-level. Modified: pypy/dist/pypy/module/struct/error.py ============================================================================== --- pypy/dist/pypy/module/struct/error.py (original) +++ pypy/dist/pypy/module/struct/error.py Thu Sep 27 10:17:38 2007 @@ -1,3 +1,4 @@ +from pypy.interpreter.error import OperationError class StructError(Exception): "Interp-level error that gets mapped to an app-level struct.error." @@ -7,3 +8,8 @@ def __str__(self): return self.msg + + def at_applevel(self, space): + w_module = space.getbuiltinmodule('struct') + w_error = space.getattr(w_module, space.wrap('error')) + return OperationError(w_error, space.wrap(self.msg)) Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Thu Sep 27 10:17:38 2007 @@ -56,7 +56,6 @@ break repetitions = ovfcheck(repetitions * 10) repetitions = ovfcheck(repetitions + (ord(c) - ord('0'))) - # XXX catch OverflowError somewhere else: repetitions = 1 @@ -68,6 +67,10 @@ break else: raise StructError("bad char in struct format") + self.finished() + + def finished(self): + pass class CalcSizeFormatIterator(FormatIterator): @@ -90,7 +93,8 @@ def __init__(self, space, args_w): self.space = space - self.args_iterator = iter(args_w) + self.args_w = args_w + self.args_index = 0 self.result = [] # list of characters def operate(self, fmtdesc, repetitions): @@ -106,12 +110,24 @@ for i in range(pad): self.result.append('\x00') + def finished(self): + if self.args_index != len(self.args_w): + raise StructError("too many arguments for struct format") + + def accept_obj_arg(self): + try: + w_obj = self.args_w[self.args_index] + except IndexError: + raise StructError("struct format requires more arguments") + self.args_index += 1 + return w_obj + def accept_int_arg(self): - w_obj = self.args_iterator.next() # XXX catch StopIteration + w_obj = self.accept_obj_arg() return self.space.int_w(w_obj) def accept_str_arg(self): - w_obj = self.args_iterator.next() # XXX catch StopIteration + w_obj = self.accept_obj_arg() return self.space.str_w(w_obj) @@ -134,10 +150,14 @@ def align(self, mask): self.inputpos = (self.inputpos + mask) & ~mask + def finished(self): + if self.inputpos != len(self.input): + raise StructError("unpack str size too long for format") + def read(self, count): end = self.inputpos + count if end > len(self.input): - raise StructError("unpack str size is too short for the format") + raise StructError("unpack str size too short for format") s = self.input[self.inputpos : end] self.inputpos = end return s Modified: pypy/dist/pypy/module/struct/interp_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/interp_struct.py (original) +++ pypy/dist/pypy/module/struct/interp_struct.py Thu Sep 27 10:17:38 2007 @@ -1,20 +1,38 @@ from pypy.interpreter.gateway import ObjSpace +from pypy.interpreter.error import OperationError +from pypy.module.struct.error import StructError from pypy.module.struct.formatiterator import CalcSizeFormatIterator from pypy.module.struct.formatiterator import PackFormatIterator from pypy.module.struct.formatiterator import UnpackFormatIterator +def overflow(space): + return OperationError(space.w_OverflowError, + space.wrap("struct format too large")) + + def calcsize(space, format): fmtiter = CalcSizeFormatIterator() - fmtiter.interpret(format) + try: + fmtiter.interpret(format) + except StructError, e: + raise e.at_applevel(space) + except OverflowError: + raise overflow(space) return space.wrap(fmtiter.totalsize) calcsize.unwrap_spec = [ObjSpace, str] def pack(space, format, args_w): fmtiter = PackFormatIterator(space, args_w) - fmtiter.interpret(format) + try: + fmtiter.interpret(format) + except StructError, e: + raise e.at_applevel(space) + except OverflowError: + raise overflow(space) # XXX check that all arguments have been consumed + "too many arguments for struct format" result = ''.join(fmtiter.result) return space.wrap(result) pack.unwrap_spec = [ObjSpace, str, 'args_w'] @@ -22,7 +40,13 @@ def unpack(space, format, input): fmtiter = UnpackFormatIterator(space, input) - fmtiter.interpret(format) + try: + fmtiter.interpret(format) + except StructError, e: + raise e.at_applevel(space) + except OverflowError: + raise overflow(space) # XXX check that the input string has been fully consumed + "unpack str size too long for format" return space.newtuple(fmtiter.result_w) unpack.unwrap_spec = [ObjSpace, str, str] Modified: pypy/dist/pypy/module/struct/nativefmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/nativefmttable.py (original) +++ pypy/dist/pypy/module/struct/nativefmttable.py Thu Sep 27 10:17:38 2007 @@ -1,5 +1,4 @@ import struct -from pypy.module.struct.error import StructError from pypy.module.struct import standardfmttable as std from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import lltype, rffi Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Thu Sep 27 10:17:38 2007 @@ -187,3 +187,53 @@ assert pack("5x") == "\x00" * 5 assert unpack("x", "?") == () assert unpack("5x", "hello") == () + + + def test_struct_error(self): + """ + Check the various ways to get a struct.error. Note that CPython + and PyPy might disagree on the specific exception raised in a + specific situation, e.g. struct.error/TypeError/OverflowError. + """ + calcsize = self.struct.calcsize + pack = self.struct.pack + unpack = self.struct.unpack + error = self.struct.error + try: + calcsize("12") # incomplete struct format + except error: # (but ignored on CPython) + pass + raises(error, calcsize, "[") # bad char in struct format + raises(error, calcsize, "!P") # bad char in struct format + raises(error, pack, "ii", 15) # struct format requires more arguments + raises(error, pack, "i", 3, 4) # too many arguments for struct format + raises(error, unpack, "ii", "?")# unpack str size too short for format + raises(error, unpack, "b", "??")# unpack str size too long for format + raises(error, pack, "c", "foo") # expected a string of length 1 + try: + pack("0p") # bad '0p' in struct format + except error: # (but ignored on CPython) + pass + try: + unpack("0p", "") # bad '0p' in struct format + except error: # (but ignored on CPython) + pass + raises(error, pack, "b", 150) # argument out of range + # XXX the accepted ranges still differs between PyPy and CPython + + + def test_overflow_error(self): + """ + Check OverflowError cases. + """ + import sys + calcsize = self.struct.calcsize + pack = self.struct.pack + unpack = self.struct.unpack + someerror = (OverflowError, self.struct.error) + raises(someerror, calcsize, "%dc" % (sys.maxint+1,)) + raises(someerror, calcsize, "999999999999999999999999999c") + raises(someerror, calcsize, "%di" % (sys.maxint,)) + raises(someerror, calcsize, "%dcc" % (sys.maxint,)) + raises(someerror, calcsize, "c%dc" % (sys.maxint,)) + raises(someerror, calcsize, "%dci" % (sys.maxint,)) From arigo at codespeak.net Thu Sep 27 10:17:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 10:17:59 +0200 (CEST) Subject: [pypy-svn] r46926 - pypy/dist/pypy/module/struct Message-ID: <20070927081759.99AAE813D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 10:17:59 2007 New Revision: 46926 Modified: pypy/dist/pypy/module/struct/interp_struct.py Log: Remove these lines, checked in by mistake. Modified: pypy/dist/pypy/module/struct/interp_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/interp_struct.py (original) +++ pypy/dist/pypy/module/struct/interp_struct.py Thu Sep 27 10:17:59 2007 @@ -31,8 +31,6 @@ raise e.at_applevel(space) except OverflowError: raise overflow(space) - # XXX check that all arguments have been consumed - "too many arguments for struct format" result = ''.join(fmtiter.result) return space.wrap(result) pack.unwrap_spec = [ObjSpace, str, 'args_w'] @@ -46,7 +44,5 @@ raise e.at_applevel(space) except OverflowError: raise overflow(space) - # XXX check that the input string has been fully consumed - "unpack str size too long for format" return space.newtuple(fmtiter.result_w) unpack.unwrap_spec = [ObjSpace, str, str] From arigo at codespeak.net Thu Sep 27 10:26:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 10:26:57 +0200 (CEST) Subject: [pypy-svn] r46928 - in pypy/dist/pypy: interpreter module/struct module/struct/test Message-ID: <20070927082657.678D380AE@code0.codespeak.net> Author: arigo Date: Thu Sep 27 10:26:57 2007 New Revision: 46928 Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/test/test_struct.py Log: Support for wider-than-int packing. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Thu Sep 27 10:26:57 2007 @@ -863,6 +863,14 @@ raise OperationError(self.w_OverflowError, self.wrap('integer too large')) + def r_ulonglong_w(self, w_obj): + bigint = self.bigint_w(w_obj) + try: + return bigint.toulonglong() + except OverflowError: + raise OperationError(self.w_OverflowError, + self.wrap('integer too large')) + class AppExecCache(SpaceCache): def build(cache, source): Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Thu Sep 27 10:26:57 2007 @@ -126,6 +126,18 @@ w_obj = self.accept_obj_arg() return self.space.int_w(w_obj) + def accept_uint_arg(self): + w_obj = self.accept_obj_arg() + return self.space.uint_w(w_obj) + + def accept_longlong_arg(self): + w_obj = self.accept_obj_arg() + return self.space.r_longlong_w(w_obj) + + def accept_ulonglong_arg(self): + w_obj = self.accept_obj_arg() + return self.space.r_ulonglong_w(w_obj) + def accept_str_arg(self): w_obj = self.accept_obj_arg() return self.space.str_w(w_obj) Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Thu Sep 27 10:26:57 2007 @@ -63,6 +63,9 @@ assert pack("' format specifier. + """ + pack = self.struct.pack + assert pack(">i", 0x41424344) == 'ABCD' + assert pack(">i", -3) == '\xff\xff\xff\xfd' + assert pack(">i", -2147483648) == '\x80\x00\x00\x00' + assert pack(">I", 0x81424344) == '\x81BCD' + assert pack(">q", 0x4142434445464748) == 'ABCDEFGH' + assert pack(">Q", 0x8142434445464748) == '\x81BCDEFGH' + + + def test_unpack_standard_big(self): + """ + Check unpacking with the '>' format specifier. + """ + unpack = self.struct.unpack + assert unpack(">i", 'ABCD') == (0x41424344,) + assert unpack(">i", '\xff\xff\xff\xfd') == (-3,) + assert unpack(">i", '\x80\x00\x00\x00') == (-2147483648,) + assert unpack(">I", '\x81BCD') == (0x81424344,) + assert unpack(">q", 'ABCDEFGH') == (0x4142434445464748,) + assert unpack(">Q", '\x81BCDEFGH') == (0x8142434445464748,) def test_calcsize_native(self): From arigo at codespeak.net Thu Sep 27 11:08:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 11:08:20 +0200 (CEST) Subject: [pypy-svn] r46929 - in pypy/dist/pypy: annotation rlib rlib/test rpython/lltypesystem rpython/lltypesystem/test rpython/test Message-ID: <20070927090820.149CB815D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 11:08:18 2007 New Revision: 46929 Modified: pypy/dist/pypy/annotation/model.py pypy/dist/pypy/rlib/rarithmetic.py pypy/dist/pypy/rlib/test/test_rarithmetic.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_lltype.py pypy/dist/pypy/rpython/test/test_rfloat.py Log: Minimal support for single-precision floats in lltype and rarithmetic. Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Thu Sep 27 11:08:18 2007 @@ -33,6 +33,7 @@ from pypy.annotation.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, base_int +from pypy.rlib.rarithmetic import r_singlefloat import inspect, weakref from sys import maxint from pypy.annotation.description import FunctionDesc @@ -154,6 +155,15 @@ def can_be_none(self): return False +class SomeSingleFloat(SomeObject): + "Stands for an r_singlefloat." + # No operation supported, not even union with a regular float + knowntype = r_singlefloat + immutable = True + + def can_be_none(self): + return False + class SomeInteger(SomeFloat): "Stands for an object which is known to be an integer." knowntype = int @@ -576,6 +586,7 @@ (s_Bool, lltype.Bool), (SomeInteger(knowntype=r_ulonglong), NUMBER), (SomeFloat(), lltype.Float), + (SomeSingleFloat(), lltype.SingleFloat), (SomeChar(), lltype.Char), (SomeUnicodeCodePoint(), lltype.UniChar), (SomeAddress(), llmemory.Address), Modified: pypy/dist/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/dist/pypy/rlib/rarithmetic.py (original) +++ pypy/dist/pypy/rlib/rarithmetic.py Thu Sep 27 11:08:18 2007 @@ -438,3 +438,77 @@ if x == 0: x = -1 return intmask(x) + +# the 'float' C type + +class r_singlefloat(object): + """A value of the C type 'float'. + + This is a single-precision floating-point number. + Regular 'float' values in Python and RPython are double-precision. + Note that we consider this as a black box for now - the only thing + you can do with it is cast it back to a regular float.""" + + def __init__(self, floatval): + import struct + # simulates the loss of precision + self._bytes = struct.pack("f", floatval) + + def __float__(self): + import struct + return struct.unpack("f", self._bytes)[0] + + def __nonzero__(self): + raise TypeError("not supported on r_singlefloat instances") + + def __cmp__(self, other): + raise TypeError("not supported on r_singlefloat instances") + + +class For_r_singlefloat_values_Entry(extregistry.ExtRegistryEntry): + _type_ = r_singlefloat + + def compute_annotation(self): + return _somesinglefloat() + +class For_r_singlefloat_type_Entry(extregistry.ExtRegistryEntry): + _about_ = r_singlefloat + + def compute_result_annotation(self, *args_s, **kwds_s): + return _somesinglefloat() + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + v, = hop.inputargs(lltype.Float) + hop.exception_cannot_occur() + # we use cast_primitive to go between Float and SingleFloat. + return hop.genop('cast_primitive', [v], + resulttype = lltype.SingleFloat) + +def _somesinglefloat(): + """Returns SomeSingleFloat(), but also lazily register the rtyping support + for SomeSingleFloat. + """ + from pypy.annotation import model as annmodel + + if 'rtyper_makerepr' not in annmodel.SomeSingleFloat.__dict__: + from pypy.rpython.lltypesystem import lltype + from pypy.rpython.rmodel import Repr + + class SingleFloatRepr(Repr): + lowleveltype = lltype.SingleFloat + + def rtype_float(self, hop): + v, = hop.inputargs(lltype.SingleFloat) + hop.exception_cannot_occur() + # we use cast_primitive to go between Float and SingleFloat. + return hop.genop('cast_primitive', [v], + resulttype = lltype.Float) + + class __extend__(annmodel.SomeSingleFloat): + def rtyper_makerepr(self, rtyper): + return SingleFloatRepr() + def rtyper_makekey(self): + return self.__class__, + + return annmodel.SomeSingleFloat() Modified: pypy/dist/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/dist/pypy/rlib/test/test_rarithmetic.py Thu Sep 27 11:08:18 2007 @@ -295,6 +295,13 @@ py.test.raises(ValueError, break_up_float, 'e') +def test_r_singlefloat(): + x = r_singlefloat(2.5) # exact number + assert float(x) == 2.5 + x = r_singlefloat(2.1) # approximate number, bits are lost + assert float(x) != 2.1 + assert abs(float(x) - 2.1) < 1E-6 + class BaseTestRarithmetic(BaseRtypingTest): def test_formatd(self): from pypy.rlib.rarithmetic import formatd Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Thu Sep 27 11:08:18 2007 @@ -1,5 +1,5 @@ import py -from pypy.rlib.rarithmetic import r_int, r_uint, intmask +from pypy.rlib.rarithmetic import r_int, r_uint, intmask, r_singlefloat from pypy.rlib.rarithmetic import r_ulonglong, r_longlong, base_int from pypy.rlib.rarithmetic import normalizedinttype from pypy.rlib.objectmodel import Symbolic @@ -584,7 +584,10 @@ SignedLongLong = build_number("SignedLongLong", r_longlong) UnsignedLongLong = build_number("UnsignedLongLong", r_ulonglong) -Float = Primitive("Float", 0.0) +Float = Primitive("Float", 0.0) # C type 'double' +SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # C type 'float' +r_singlefloat._TYPE = SingleFloat + Char = Primitive("Char", '\x00') Bool = Primitive("Bool", False) Void = Primitive("Void", None) @@ -680,12 +683,16 @@ if ORIG == Char or ORIG == UniChar: value = ord(value) elif ORIG == Float: + if TGT == SingleFloat: + return r_singlefloat(value) value = long(value) cast = _to_primitive.get(TGT) if cast is not None: return cast(value) if isinstance(TGT, Number): return TGT._cast(value) + if ORIG == SingleFloat and TGT == Float: + return float(value) raise TypeError, "unsupported cast" def _cast_whatever(TGT, value): Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Sep 27 11:08:18 2007 @@ -254,9 +254,13 @@ # (use SIGNEDCHAR or UCHAR for the small integer types) CHAR = lltype.Char -# double - XXX there is no support for the C type 'float' in the C backend yet +# double DOUBLE = lltype.Float +# float - corresponds to pypy.rlib.rarithmetic.r_float, and supports no +# operation except rffi.cast() between FLOAT and DOUBLE +FLOAT = lltype.SingleFloat + # void * - for now, represented as char * VOIDP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_lltype.py Thu Sep 27 11:08:18 2007 @@ -549,6 +549,7 @@ def test_cast_primitive(): cases = [ (Float, 1, 1.0), + (Float, r_singlefloat(2.1), float(r_singlefloat(2.1))), (Signed, 1.0, 1), (Unsigned, 1.0, 1), (Signed, r_uint(-1), -1), @@ -562,6 +563,9 @@ res = cast_primitive(TGT, orig_val) assert typeOf(res) == TGT assert res == expect + res = cast_primitive(SingleFloat, 2.1) + assert isinstance(res, r_singlefloat) + assert float(res) == float(r_singlefloat(2.1)) def test_cast_identical_array_ptr_types(): A = GcArray(Signed) Modified: pypy/dist/pypy/rpython/test/test_rfloat.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rfloat.py (original) +++ pypy/dist/pypy/rpython/test/test_rfloat.py Thu Sep 27 11:08:18 2007 @@ -2,7 +2,7 @@ from pypy.translator.translator import TranslationContext from pypy.rpython.test import snippet from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.rlib.rarithmetic import r_uint, r_longlong +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_singlefloat class TestSnippet(object): @@ -94,6 +94,15 @@ res = self.interpret(fn, [-9]) assert self.float_eq(res, 0.5 * ((sys.maxint+1)*2 - 9)) + def test_r_singlefloat(self): + def fn(x): + y = r_singlefloat(x) + return float(y) + + res = self.interpret(fn, [2.1]) + assert res != 2.1 # precision lost + assert abs(res - 2.1) < 1E-6 + def test_float_constant_conversions(self): DIV = r_longlong(10 ** 10) def fn(): From arigo at codespeak.net Thu Sep 27 11:14:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 11:14:09 +0200 (CEST) Subject: [pypy-svn] r46930 - in pypy/dist/pypy/annotation: . test Message-ID: <20070927091409.B0B8F8170@code0.codespeak.net> Author: arigo Date: Thu Sep 27 11:14:09 2007 New Revision: 46930 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: Annotation test and fix. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Thu Sep 27 11:14:09 2007 @@ -13,7 +13,7 @@ from pypy.annotation.model import SomePBC, SomeSlice, SomeFloat, s_None from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeCTypesObject +from pypy.annotation.model import SomeCTypesObject, SomeSingleFloat from pypy.annotation.model import unionof, UnionError, set, missing_operation, TLS from pypy.annotation.model import read_can_only_throw from pypy.annotation.model import add_knowntypedata, merge_knowntypedata @@ -448,6 +448,12 @@ inplace_truediv = truediv +class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): + + def union((flt1, flt2)): + return SomeSingleFloat() + + class __extend__(pairtype(SomeList, SomeList)): def union((lst1, lst2)): Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Thu Sep 27 11:14:09 2007 @@ -14,6 +14,7 @@ from pypy.annotation.dictdef import DictDef from pypy.objspace.flow.model import * from pypy.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong +from pypy.rlib.rarithmetic import r_singlefloat from pypy.rlib import objectmodel from pypy.objspace.flow import FlowObjSpace @@ -2848,6 +2849,17 @@ a = self.RPythonAnnotator() a.build_types(f, [int, int]) + def test_r_singlefloat(self): + z = r_singlefloat(0.4) + def g(n): + if n > 0: + return r_singlefloat(n * 0.1) + else: + return z + a = self.RPythonAnnotator() + s = a.build_types(g, [int]) + assert isinstance(s, annmodel.SomeSingleFloat) + def g(n): return [0,1,2,n] From arigo at codespeak.net Thu Sep 27 11:17:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 11:17:39 +0200 (CEST) Subject: [pypy-svn] r46931 - in pypy/dist/pypy/translator: . c c/test Message-ID: <20070927091739.3106B815E@code0.codespeak.net> Author: arigo Date: Thu Sep 27 11:17:38 2007 New Revision: 46931 Modified: pypy/dist/pypy/translator/c/primitive.py pypy/dist/pypy/translator/c/test/test_lltyped.py pypy/dist/pypy/translator/exceptiontransform.py Log: Float support in the exception transformer and C backend. Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Thu Sep 27 11:17:38 2007 @@ -97,6 +97,19 @@ else: return repr(value) +def name_singlefloat(value, db): + value = float(value) + if isinf(value): + if value > 0: + return '((float)Py_HUGE_VAL)' + else: + return '((float)-Py_HUGE_VAL)' + elif isnan(value): + # XXX are these expressions ok? + return '((float)(Py_HUGE_VAL/Py_HUGE_VAL))' + else: + return repr(value) + 'f' + def name_char(value, db): assert type(value) is str and len(value) == 1 if ' ' <= value < '\x7f': @@ -128,6 +141,7 @@ UnsignedLongLong: name_unsignedlonglong, Unsigned: name_unsigned, Float: name_float, + SingleFloat: name_singlefloat, Char: name_char, UniChar: name_unichar, Bool: name_bool, @@ -141,6 +155,7 @@ UnsignedLongLong: 'unsigned long long @', Unsigned: 'unsigned long @', Float: 'double @', + SingleFloat: 'float @', Char: 'char @', UniChar: 'unsigned int @', Bool: 'bool_t @', @@ -154,6 +169,7 @@ UnsignedLongLong: '((unsigned long long) -1)', Unsigned: '((unsigned) -1)', Float: '-1.0', + SingleFloat: '-1.0f', Char: '((char) -1)', UniChar: '((unsigned) -1)', Bool: '0 /* error */', Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Thu Sep 27 11:17:38 2007 @@ -598,3 +598,24 @@ fn = self.getcompiled(llf) fn() + + def test_r_singlefloat(self): + + z = r_singlefloat(0.4) + + def g(n): + if n > 0: + return r_singlefloat(n * 0.1) + else: + return z + + def llf(n): + return float(g(n)) + + fn = self.getcompiled(llf, [int]) + res = fn(21) + assert res != 2.1 # precision lost + assert abs(res - 2.1) < 1E-6 + res = fn(-5) + assert res != 0.4 # precision lost + assert abs(res - 0.4) < 1E-6 Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Thu Sep 27 11:17:38 2007 @@ -12,6 +12,7 @@ from pypy.rpython import rclass from pypy.rpython.rmodel import inputconst from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong +from pypy.rlib.rarithmetic import r_singlefloat from pypy.annotation import model as annmodel from pypy.rpython.annlowlevel import MixLevelHelperAnnotator @@ -20,6 +21,7 @@ lltype.SignedLongLong: r_longlong(-1), lltype.UnsignedLongLong: r_ulonglong(-1), lltype.Float: -1.0, + lltype.SingleFloat: r_singlefloat(-1.0), lltype.Char: chr(255), lltype.UniChar: unichr(0xFFFF), # XXX is this always right? lltype.Bool: True, From arigo at codespeak.net Thu Sep 27 11:29:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 11:29:34 +0200 (CEST) Subject: [pypy-svn] r46932 - in pypy/dist/pypy: interpreter/test objspace/std objspace/std/test Message-ID: <20070927092934.7536C8172@code0.codespeak.net> Author: arigo Date: Thu Sep 27 11:29:34 2007 New Revision: 46932 Modified: pypy/dist/pypy/interpreter/test/test_objspace.py pypy/dist/pypy/objspace/std/default.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/test/test_longobject.py Log: Missing the stub to raise an app-level TypeError for two of the multimethods. Modified: pypy/dist/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_objspace.py (original) +++ pypy/dist/pypy/interpreter/test/test_objspace.py Thu Sep 27 11:29:34 2007 @@ -166,14 +166,19 @@ assert 0, "should have raised" def test_r_longlong_w(self): - w_value = self.space.wrap(12) - res = self.space.r_longlong_w(w_value) + space = self.space + w_value = space.wrap(12) + res = space.r_longlong_w(w_value) assert res == 12 assert type(res) is r_longlong - w_value = self.space.wrap(r_longlong(-sys.maxint * 42)) - res = self.space.r_longlong_w(w_value) + w_value = space.wrap(r_longlong(-sys.maxint * 42)) + res = space.r_longlong_w(w_value) assert res == -sys.maxint * 42 assert type(res) is r_longlong + w_obj = space.wrap("hello world") + space.raises_w(space.w_TypeError, space.r_longlong_w, w_obj) + w_obj = space.wrap(-12.34) + space.raises_w(space.w_TypeError, space.r_longlong_w, w_obj) class TestModuleMinimal: Modified: pypy/dist/pypy/objspace/std/default.py ============================================================================== --- pypy/dist/pypy/objspace/std/default.py (original) +++ pypy/dist/pypy/objspace/std/default.py Thu Sep 27 11:29:34 2007 @@ -35,4 +35,12 @@ raise OperationError(space.w_TypeError, typed_unwrap_error_msg(space, "integer", w_obj)) +def unichars_w__ANY(space,w_obj): + raise OperationError(space.w_TypeError, + typed_unwrap_error_msg(space, "string", w_obj)) + +def bigint_w__ANY(space,w_obj): + raise OperationError(space.w_TypeError, + typed_unwrap_error_msg(space, "integer", w_obj)) + register_all(vars()) Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Thu Sep 27 11:29:34 2007 @@ -643,6 +643,8 @@ uint_w = StdObjSpaceMultiMethod('uint_w', 1, []) # returns an unwrapped unsigned int (r_uint) unichars_w = StdObjSpaceMultiMethod('unichars_w', 1, []) # returns an unwrapped list of unicode characters bigint_w = StdObjSpaceMultiMethod('bigint_w', 1, []) # returns an unwrapped rbigint + # NOTE: when adding more sometype_w() methods, you need to write a + # stub in default.py to raise a space.w_TypeError marshal_w = StdObjSpaceMultiMethod('marshal_w', 1, [], extra_args=['marshaller']) log = StdObjSpaceMultiMethod('log', 1, [], extra_args=['base']) Modified: pypy/dist/pypy/objspace/std/test/test_longobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_longobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_longobject.py Thu Sep 27 11:29:34 2007 @@ -14,6 +14,10 @@ assert isinstance(space.bigint_w(fromlong(42)), rbigint) assert space.bigint_w(fromlong(42)).eq(rbigint.fromint(42)) assert space.bigint_w(fromlong(-1)).eq(rbigint.fromint(-1)) + w_obj = space.wrap("hello world") + space.raises_w(space.w_TypeError, space.bigint_w, w_obj) + w_obj = space.wrap(123.456) + space.raises_w(space.w_TypeError, space.bigint_w, w_obj) class AppTestLong: def test_add(self): From cfbolz at codespeak.net Thu Sep 27 11:35:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 27 Sep 2007 11:35:24 +0200 (CEST) Subject: [pypy-svn] r46933 - pypy/dist/pypy/doc/config Message-ID: <20070927093524.E28C08171@code0.codespeak.net> Author: cfbolz Date: Thu Sep 27 11:35:24 2007 New Revision: 46933 Removed: pypy/dist/pypy/doc/config/objspace.std.oldstyle.txt Log: remove docs for the now-removed --oldstyle config option From cfbolz at codespeak.net Thu Sep 27 11:37:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 27 Sep 2007 11:37:36 +0200 (CEST) Subject: [pypy-svn] r46934 - pypy/dist/pypy/doc/config Message-ID: <20070927093736.F09668171@code0.codespeak.net> Author: cfbolz Date: Thu Sep 27 11:37:36 2007 New Revision: 46934 Added: pypy/dist/pypy/doc/config/objspace.usemodules.zlib.txt Log: add description of zlib module Added: pypy/dist/pypy/doc/config/objspace.usemodules.zlib.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.usemodules.zlib.txt Thu Sep 27 11:37:36 2007 @@ -0,0 +1,2 @@ +Use the 'zlib' module. +This module is expected to be working and is included by default. From arigo at codespeak.net Thu Sep 27 11:48:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 11:48:35 +0200 (CEST) Subject: [pypy-svn] r46937 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070927094835.B753E8172@code0.codespeak.net> Author: arigo Date: Thu Sep 27 11:48:35 2007 New Revision: 46937 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: More singlefloat support in rffi and ll2ctypes. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Thu Sep 27 11:48:35 2007 @@ -12,7 +12,7 @@ from pypy.rlib.objectmodel import Symbolic from pypy.tool.uid import fixid from pypy.tool.tls import tlsobject -from pypy.rlib.rarithmetic import r_uint +from pypy.rlib.rarithmetic import r_uint, r_singlefloat from pypy.annotation import model as annmodel from pypy.rpython.rbuiltin import gen_cast @@ -30,6 +30,7 @@ lltype.Unsigned: ctypes.c_ulong, lltype.Char: ctypes.c_ubyte, rffi.DOUBLE: ctypes.c_double, + rffi.FLOAT: ctypes.c_float, rffi.SIGNEDCHAR: ctypes.c_byte, rffi.UCHAR: ctypes.c_ubyte, rffi.SHORT: ctypes.c_short, @@ -428,6 +429,9 @@ if T is lltype.Char: return ord(llobj) + if T is lltype.SingleFloat: + return ctypes.c_float(float(llobj)) + return llobj def ctypes2lltype(T, cobj): @@ -459,7 +463,11 @@ llobj = lltype._ptr(T, container, solid=True) elif T is lltype.Char: llobj = chr(cobj) - elif T is not lltype.Signed: + elif T is lltype.Signed: + llobj = cobj + elif T is lltype.SingleFloat: + llobj = r_singlefloat(cobj.value) + else: from pypy.rpython.lltypesystem import rffi try: inttype = rffi.platform.numbertype_to_rclass[T] @@ -467,8 +475,6 @@ llobj = cobj else: llobj = inttype(cobj) - else: - llobj = cobj assert lltype.typeOf(llobj) == T return llobj Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Sep 27 11:48:35 2007 @@ -260,6 +260,7 @@ # float - corresponds to pypy.rlib.rarithmetic.r_float, and supports no # operation except rffi.cast() between FLOAT and DOUBLE FLOAT = lltype.SingleFloat +r_singlefloat = rarithmetic.r_singlefloat # void * - for now, represented as char * VOIDP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) @@ -273,6 +274,9 @@ # double * DOUBLEP = lltype.Ptr(lltype.Array(DOUBLE, hints={'nolength': True})) +# float * +FLOATP = lltype.Ptr(lltype.Array(FLOAT, hints={'nolength': True})) + # various type mapping # str -> char* def str2charp(s): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu Sep 27 11:48:35 2007 @@ -23,6 +23,12 @@ assert ctypes2lltype(lltype.Char, 0xFF) == '\xFF' assert lltype2ctypes(5.25) == 5.25 assert ctypes2lltype(lltype.Float, 5.25) == 5.25 + res = lltype2ctypes(rffi.r_singlefloat(-3.5)) + assert isinstance(res, ctypes.c_float) + assert res.value == -3.5 + res = ctypes2lltype(lltype.SingleFloat, ctypes.c_float(-3.5)) + assert isinstance(res, rffi.r_singlefloat) + assert float(res) == -3.5 assert lltype2ctypes(rffi.r_uint(-1)) == sys.maxint * 2 + 1 res = ctypes2lltype(lltype.Unsigned, sys.maxint * 2 + 1) assert (res, type(res)) == (rffi.r_uint(-1), rffi.r_uint) From arigo at codespeak.net Thu Sep 27 11:53:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 11:53:27 +0200 (CEST) Subject: [pypy-svn] r46938 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20070927095327.ED012816B@code0.codespeak.net> Author: arigo Date: Thu Sep 27 11:53:27 2007 New Revision: 46938 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: More tests and fixes. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Thu Sep 27 11:53:27 2007 @@ -466,7 +466,9 @@ elif T is lltype.Signed: llobj = cobj elif T is lltype.SingleFloat: - llobj = r_singlefloat(cobj.value) + if isinstance(cobj, ctypes.c_float): + cobj = cobj.value + llobj = r_singlefloat(cobj) else: from pypy.rpython.lltypesystem import rffi try: Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu Sep 27 11:53:27 2007 @@ -631,3 +631,20 @@ lltype.free(s.a, flavor='raw') lltype.free(s, flavor='raw') assert not ALLOCATED # detects memory leaks in the test + + def test_arrayoffloat(self): + a = lltype.malloc(rffi.FLOATP.TO, 3, flavor='raw') + a[0] = rffi.r_singlefloat(0.0) + a[1] = rffi.r_singlefloat(1.1) + a[2] = rffi.r_singlefloat(2.2) + ac = lltype2ctypes(a, normalize=False) + assert ac.contents.items[0] == 0.0 + assert abs(ac.contents.items[1] - 1.1) < 1E-6 + assert abs(ac.contents.items[2] - 2.2) < 1E-6 + b = ctypes2lltype(rffi.FLOATP, ac) + assert isinstance(b[0], rffi.r_singlefloat) + assert float(b[0]) == 0.0 + assert isinstance(b[1], rffi.r_singlefloat) + assert abs(float(b[1]) - 1.1) < 1E-6 + assert isinstance(b[2], rffi.r_singlefloat) + assert abs(float(b[2]) - 2.2) < 1E-6 From arigo at codespeak.net Thu Sep 27 11:54:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 11:54:15 +0200 (CEST) Subject: [pypy-svn] r46939 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070927095415.8F9498170@code0.codespeak.net> Author: arigo Date: Thu Sep 27 11:54:15 2007 New Revision: 46939 Modified: pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/nativefmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: Native floats and doubles in the struct module. Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Thu Sep 27 11:54:15 2007 @@ -142,6 +142,10 @@ w_obj = self.accept_obj_arg() return self.space.str_w(w_obj) + def accept_float_arg(self): + w_obj = self.accept_obj_arg() + return self.space.float_w(w_obj) + class UnpackFormatIterator(FormatIterator): Modified: pypy/dist/pypy/module/struct/nativefmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/nativefmttable.py (original) +++ pypy/dist/pypy/module/struct/nativefmttable.py Thu Sep 27 11:54:15 2007 @@ -2,6 +2,7 @@ from pypy.module.struct import standardfmttable as std from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.rarithmetic import r_singlefloat native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1) @@ -13,49 +14,113 @@ } # ____________________________________________________________ + +def pack_double(fmtiter): + doubleval = fmtiter.accept_float_arg() + buf = lltype.malloc(rffi.DOUBLEP.TO, 1, flavor='raw') + try: + buf[0] = doubleval + p = rffi.cast(rffi.CCHARP, buf) + for i in range(sizeof_double): + fmtiter.result.append(p[i]) + finally: + lltype.free(buf, flavor='raw') + +def unpack_double(fmtiter): + input = fmtiter.read(sizeof_double) + buf = lltype.malloc(rffi.DOUBLEP.TO, 1, flavor='raw') + try: + p = rffi.cast(rffi.CCHARP, buf) + for i in range(sizeof_double): + p[i] = input[i] + doubleval = buf[0] + finally: + lltype.free(buf, flavor='raw') + fmtiter.appendobj(doubleval) + +def pack_float(fmtiter): + doubleval = fmtiter.accept_float_arg() + floatval = r_singlefloat(doubleval) + buf = lltype.malloc(rffi.FLOATP.TO, 1, flavor='raw') + try: + buf[0] = floatval + p = rffi.cast(rffi.CCHARP, buf) + for i in range(sizeof_float): + fmtiter.result.append(p[i]) + finally: + lltype.free(buf, flavor='raw') + +def unpack_float(fmtiter): + input = fmtiter.read(sizeof_float) + buf = lltype.malloc(rffi.FLOATP.TO, 1, flavor='raw') + try: + p = rffi.cast(rffi.CCHARP, buf) + for i in range(sizeof_float): + p[i] = input[i] + floatval = buf[0] + finally: + lltype.free(buf, flavor='raw') + doubleval = float(floatval) + fmtiter.appendobj(doubleval) + +# ____________________________________________________________ # # Use rffi_platform to get the native sizes and alignments from the C compiler -INSPECT = {'b': 'signed char', - 'h': 'signed short', - 'i': 'signed int', - 'l': 'signed long', - 'q': 'signed long long', - 'B': 'unsigned char', - 'H': 'unsigned short', - 'I': 'unsigned int', - 'L': 'unsigned long', - 'Q': 'unsigned long long', - 'P': 'char *', - 'f': 'float', - 'd': 'double', - } - -class CConfig: - _header_ = "" - -for fmtchar, ctype in INSPECT.items(): - CConfig._header_ += """ - struct about_%s { - char pad; - %s field; - }; - """ % (fmtchar, ctype) - setattr(CConfig, fmtchar, rffi_platform.Struct( - "struct about_%s" % (fmtchar,), - [('field', lltype.FixedSizeArray(rffi.CHAR, 1))])) - -cConfig = rffi_platform.configure(CConfig) - -for fmtchar, ctype in INSPECT.items(): - S = cConfig[fmtchar] - alignment = rffi.offsetof(S, 'c_field') - size = rffi.sizeof(S.c_field) - signed = 'a' <= fmtchar <= 'z' - - native_fmttable[fmtchar] = { - 'size': size, - 'alignment': alignment, - 'pack': std.make_int_packer(size, signed), # XXX 'f', 'd' - 'unpack': std.make_int_unpacker(size, signed), - } +def setup(): + INSPECT = {'b': 'signed char', + 'h': 'signed short', + 'i': 'signed int', + 'l': 'signed long', + 'q': 'signed long long', + 'B': 'unsigned char', + 'H': 'unsigned short', + 'I': 'unsigned int', + 'L': 'unsigned long', + 'Q': 'unsigned long long', + 'P': 'char *', + 'f': 'float', + 'd': 'double', + } + + class CConfig: + _header_ = "" + + for fmtchar, ctype in INSPECT.items(): + CConfig._header_ += """ + struct about_%s { + char pad; + %s field; + }; + """ % (fmtchar, ctype) + setattr(CConfig, fmtchar, rffi_platform.Struct( + "struct about_%s" % (fmtchar,), + [('field', lltype.FixedSizeArray(rffi.CHAR, 1))])) + + cConfig = rffi_platform.configure(CConfig) + + for fmtchar, ctype in INSPECT.items(): + S = cConfig[fmtchar] + alignment = rffi.offsetof(S, 'c_field') + size = rffi.sizeof(S.c_field) + signed = 'a' <= fmtchar <= 'z' + + if fmtchar == 'f': + pack = pack_float + unpack = unpack_float + elif fmtchar == 'd': + pack = pack_double + unpack = unpack_double + else: + pack = std.make_int_packer(size, signed) + unpack = std.make_int_unpacker(size, signed) + + native_fmttable[fmtchar] = {'size': size, + 'alignment': alignment, + 'pack': pack, + 'unpack': unpack} + +setup() + +sizeof_double = native_fmttable['d']['size'] +sizeof_float = native_fmttable['f']['size'] Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Thu Sep 27 11:54:15 2007 @@ -221,6 +221,37 @@ assert unpack("5x", "hello") == () + def test_native_floats(self): + """ + Check the 'd' and 'f' format characters on native packing. + """ + calcsize = self.struct.calcsize + pack = self.struct.pack + unpack = self.struct.unpack + data = pack("d", 12.34) + assert len(data) == calcsize("d") + assert unpack("d", data) == (12.34,) # no precision lost + data = pack("f", 12.34) + assert len(data) == calcsize("f") + res, = unpack("f", data) + assert res != 12.34 # precision lost + assert abs(res - 12.34) < 1E-6 + + + def test_standard_floats(self): + """ + Check the 'd' and 'f' format characters on standard packing. + """ + skip("in-progress") + pack = self.struct.pack + unpack = self.struct.unpack + assert pack("!d", 12.5) == '@)\x00\x00\x00\x00\x00\x00' + assert pack(" Author: niko Date: Thu Sep 27 12:10:30 2007 New Revision: 46942 Modified: pypy/extradoc/sprintinfo/bern2007/people.txt Log: add myself to the list with expected dates Modified: pypy/extradoc/sprintinfo/bern2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/people.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/people.txt Thu Sep 27 12:10:30 2007 @@ -10,6 +10,7 @@ ==================== ============== ===================== Carl Friedrich Bolz ? ? Adrian Kuhn - private +Niko Matsakis 22.10 / 26.10 private (Zurich) ==================== ============== ===================== @@ -27,7 +28,6 @@ Andrew Thompson ? ? Stephan Diehl ? ? Eric van Riet paap ? ? -Niko Matsakis ? ? Leonardo Santagada ? ? Richard Emslie ? ? Christian Tismer ? ? From arigo at codespeak.net Thu Sep 27 13:09:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 13:09:48 +0200 (CEST) Subject: [pypy-svn] r46943 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070927110948.AD7EA818C@code0.codespeak.net> Author: arigo Date: Thu Sep 27 13:09:47 2007 New Revision: 46943 Added: pypy/dist/pypy/module/struct/ieee.py - copied, changed from r46897, pypy/dist/pypy/lib/struct.py pypy/dist/pypy/module/struct/test/test_ieee.py (contents, props changed) Log: Copy this code from pypy/lib/struct and RPythonify it. Added: pypy/dist/pypy/module/struct/test/test_ieee.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/struct/test/test_ieee.py Thu Sep 27 13:09:47 2007 @@ -0,0 +1,30 @@ +from pypy.module.struct.ieee import pack_float, unpack_float + + +testcases = [ + (-0.025, 4, False, '\xcd\xcc\xcc\xbc'), + (2.0 ** 100, 4, False, '\x00\x00\x80q'), + (0.0, 4, True, '\x00\x00\x00\x00'), + (-12345, 4, True, '\xc6@\xe4\x00'), + (-0.025, 8, False, '\x9a\x99\x99\x99\x99\x99\x99\xbf'), + (2.0 ** 100, 8, False, '\x00\x00\x00\x00\x00\x000F'), + (0.0, 8, True, '\x00\x00\x00\x00\x00\x00\x00\x00'), + (-123456789, 8, True, '\xc1\x9do4T\x00\x00\x00'), + ] + + +def test_pack(): + for number, size, bigendian, expected in testcases: + res = [] + pack_float(res, number, size, bigendian) + assert res == list(expected) + + +def test_unpack(): + for expected, size, bigendian, input in testcases: + assert len(input) == size + res = unpack_float(input, bigendian) + if size == 8: + assert res == expected # exact result expected + else: + assert abs(res - expected) < 1E-6 From arigo at codespeak.net Thu Sep 27 13:10:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 13:10:37 +0200 (CEST) Subject: [pypy-svn] r46944 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070927111037.E1FB3818D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 13:10:37 2007 New Revision: 46944 Modified: pypy/dist/pypy/module/struct/standardfmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: Use ieee.py to implement the 'f' and 'd' format characters in standard sizes. Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Thu Sep 27 13:10:37 2007 @@ -7,6 +7,7 @@ import struct from pypy.module.struct.error import StructError +from pypy.module.struct import ieee from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong @@ -48,8 +49,11 @@ for i in range(1 + prefix, count): fmtiter.result.append('\x00') -def pack_float(fmtiter): - xxx +def make_float_packer(size): + return lambda fmtiter: ieee.pack_float(fmtiter.result, + fmtiter.accept_float_arg(), + size, + fmtiter.bigendian) # ____________________________________________________________ @@ -127,8 +131,10 @@ end = count fmtiter.appendobj(data[1:end]) -def unpack_float(fmtiter): - xxx +def make_float_unpacker(size): + return lambda fmtiter: fmtiter.appendobj(ieee.unpack_float( + fmtiter.read(size), + fmtiter.bigendian)) # ____________________________________________________________ @@ -185,8 +191,10 @@ 'needcount' : True }, 'p':{ 'size' : 1, 'pack' : pack_pascal, 'unpack' : unpack_pascal, 'needcount' : True }, - 'f':{ 'size' : 4, 'pack' : pack_float, 'unpack' : unpack_float}, - 'd':{ 'size' : 8, 'pack' : pack_float, 'unpack' : unpack_float}, + 'f':{ 'size' : 4, 'pack' : make_float_packer(4), + 'unpack' : make_float_unpacker(4)}, + 'd':{ 'size' : 8, 'pack' : make_float_packer(8), + 'unpack' : make_float_unpacker(8)}, } for c, size in [('b', 1), ('h', 2), ('i', 4), ('l', 4), ('q', 8)]: Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Thu Sep 27 13:10:37 2007 @@ -242,14 +242,16 @@ """ Check the 'd' and 'f' format characters on standard packing. """ - skip("in-progress") pack = self.struct.pack unpack = self.struct.unpack assert pack("!d", 12.5) == '@)\x00\x00\x00\x00\x00\x00' - assert pack(" Author: arigo Date: Thu Sep 27 13:16:51 2007 New Revision: 46945 Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/test/test_objspace.py Log: Report this ValueError to app-level. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Thu Sep 27 13:16:51 2007 @@ -870,6 +870,10 @@ except OverflowError: raise OperationError(self.w_OverflowError, self.wrap('integer too large')) + except ValueError: + raise OperationError(self.w_ValueError, + self.wrap('cannot convert negative integer ' + 'to unsigned int')) class AppExecCache(SpaceCache): Modified: pypy/dist/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_objspace.py (original) +++ pypy/dist/pypy/interpreter/test/test_objspace.py Thu Sep 27 13:16:51 2007 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.function import Function from pypy.interpreter.pycode import PyCode -from pypy.rlib.rarithmetic import r_longlong +from pypy.rlib.rarithmetic import r_longlong, r_ulonglong import sys # this test isn't so much to test that the objspace interface *works* @@ -180,6 +180,23 @@ w_obj = space.wrap(-12.34) space.raises_w(space.w_TypeError, space.r_longlong_w, w_obj) + def test_r_ulonglong_w(self): + space = self.space + w_value = space.wrap(12) + res = space.r_ulonglong_w(w_value) + assert res == 12 + assert type(res) is r_ulonglong + w_value = space.wrap(r_ulonglong(sys.maxint * 42)) + res = space.r_ulonglong_w(w_value) + assert res == sys.maxint * 42 + assert type(res) is r_ulonglong + w_obj = space.wrap("hello world") + space.raises_w(space.w_TypeError, space.r_ulonglong_w, w_obj) + w_obj = space.wrap(-12.34) + space.raises_w(space.w_TypeError, space.r_ulonglong_w, w_obj) + w_obj = space.wrap(-12) + space.raises_w(space.w_ValueError, space.r_ulonglong_w, w_obj) + class TestModuleMinimal: def test_sys_exists(self): From arigo at codespeak.net Thu Sep 27 13:30:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 13:30:12 +0200 (CEST) Subject: [pypy-svn] r46946 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070927113012.C055380A7@code0.codespeak.net> Author: arigo Date: Thu Sep 27 13:30:12 2007 New Revision: 46946 Modified: pypy/dist/pypy/module/struct/standardfmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: bug found by CPython's test_struct. Test and fix. Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Thu Sep 27 13:30:12 2007 @@ -144,13 +144,11 @@ except KeyError: pass if signed: - max = (2 ** (8*size-1)) - 1 if size <= native_int_size: inttype = int else: inttype = r_longlong else: - max = None if size < native_int_size: inttype = int elif size == native_int_size: @@ -165,17 +163,19 @@ idx = 0 if fmtiter.bigendian: for i in unroll_range_size: + x = ord(s[idx]) + if signed and i == 0 and x >= 128: + x -= 256 intvalue <<= 8 - intvalue |= ord(s[idx]) + intvalue |= x idx += 1 else: for i in unroll_range_size: - intvalue |= inttype(ord(s[idx])) << (8*i) + x = ord(s[idx]) + if signed and i == size - 1 and x >= 128: + x -= 256 + intvalue |= inttype(x) << (8*i) idx += 1 - if max is not None and intvalue > max: - intvalue -= max - intvalue -= max - intvalue -= 2 fmtiter.appendobj(intvalue) _memo[size, signed] = unpack_int Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Thu Sep 27 13:30:12 2007 @@ -65,6 +65,7 @@ assert pack("i", -2147483648) == '\x80\x00\x00\x00' assert pack(">I", 0x81424344) == '\x81BCD' assert pack(">q", 0x4142434445464748) == 'ABCDEFGH' + assert pack(">q", -0x41B2B3B4B5B6B7B8) == '\xbeMLKJIHH' assert pack(">Q", 0x8142434445464748) == '\x81BCDEFGH' @@ -104,6 +107,7 @@ assert unpack(">i", '\x80\x00\x00\x00') == (-2147483648,) assert unpack(">I", '\x81BCD') == (0x81424344,) assert unpack(">q", 'ABCDEFGH') == (0x4142434445464748,) + assert unpack(">q", '\xbeMLKJIHH') == (-0x41B2B3B4B5B6B7B8,) assert unpack(">Q", '\x81BCDEFGH') == (0x8142434445464748,) @@ -141,8 +145,9 @@ """ Check packing with the native format. """ + calcsize = self.struct.calcsize pack = self.struct.pack - sizeofi = self.struct.calcsize("i") + sizeofi = calcsize("i") res = pack("bi", -2, 5) assert len(res) == 2 * sizeofi assert res[0] == '\xfe' @@ -151,15 +156,18 @@ assert res[sizeofi:] == '\x00' * (sizeofi-1) + '\x05' else: assert res[sizeofi:] == '\x05' + '\x00' * (sizeofi-1) + assert pack("q", -1) == '\xff' * calcsize("q") def test_unpack_native(self): """ Check unpacking with the native format. """ + calcsize = self.struct.calcsize pack = self.struct.pack unpack = self.struct.unpack assert unpack("bi", pack("bi", -2, 5)) == (-2, 5) + assert unpack("q", '\xff' * calcsize("q")) == (-1,) def test_string_format(self): From arigo at codespeak.net Thu Sep 27 13:32:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 13:32:39 +0200 (CEST) Subject: [pypy-svn] r46947 - pypy/dist/pypy/module/struct Message-ID: <20070927113239.F01D6818F@code0.codespeak.net> Author: arigo Date: Thu Sep 27 13:32:39 2007 New Revision: 46947 Modified: pypy/dist/pypy/module/struct/formatiterator.py Log: Translation fix. Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Thu Sep 27 13:32:39 2007 @@ -82,7 +82,7 @@ else: size = ovfcheck(fmtdesc.size * repetitions) self.totalsize = ovfcheck(self.totalsize + size) - operate._annspecialcase_ = 'specialize:argvalue(1)' + operate._annspecialcase_ = 'specialize:arg(1)' def align(self, mask): pad = (-self.totalsize) & mask @@ -103,7 +103,7 @@ else: for i in range(repetitions): fmtdesc.pack(self) - operate._annspecialcase_ = 'specialize:argvalue(1)' + operate._annspecialcase_ = 'specialize:arg(1)' def align(self, mask): pad = (-len(self.result)) & mask @@ -161,7 +161,7 @@ else: for i in range(repetitions): fmtdesc.unpack(self) - operate._annspecialcase_ = 'specialize:argvalue(1)' + operate._annspecialcase_ = 'specialize:arg(1)' def align(self, mask): self.inputpos = (self.inputpos + mask) & ~mask From arigo at codespeak.net Thu Sep 27 14:14:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 14:14:27 +0200 (CEST) Subject: [pypy-svn] r46949 - pypy/dist/pypy/module/struct Message-ID: <20070927121427.76D778142@code0.codespeak.net> Author: arigo Date: Thu Sep 27 14:14:26 2007 New Revision: 46949 Modified: pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/interp_struct.py Log: Move the "except OverflowError" directly around the ovfcheck(), as needed for RPython for now. Also raise struct.error instead of OverflowError at app-level, which is more in line with CPython. Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Thu Sep 27 14:14:26 2007 @@ -54,8 +54,13 @@ index += 1 if not c.isdigit(): break - repetitions = ovfcheck(repetitions * 10) - repetitions = ovfcheck(repetitions + (ord(c) - ord('0'))) + try: + repetitions = ovfcheck(repetitions * 10) + repetitions = ovfcheck(repetitions + (ord(c) - + ord('0'))) + except OverflowError: + raise StructError("overflow in item count") + assert repetitions >= 0 else: repetitions = 1 @@ -77,16 +82,23 @@ totalsize = 0 def operate(self, fmtdesc, repetitions): - if fmtdesc.size == 1: - size = repetitions # skip the overflow-checked multiplication by 1 - else: - size = ovfcheck(fmtdesc.size * repetitions) - self.totalsize = ovfcheck(self.totalsize + size) + try: + if fmtdesc.size == 1: + # skip the overflow-checked multiplication by 1 + size = repetitions + else: + size = ovfcheck(fmtdesc.size * repetitions) + self.totalsize = ovfcheck(self.totalsize + size) + except OverflowError: + raise StructError("total struct size too long") operate._annspecialcase_ = 'specialize:arg(1)' def align(self, mask): pad = (-self.totalsize) & mask - self.totalsize = ovfcheck(self.totalsize + pad) + try: + self.totalsize = ovfcheck(self.totalsize + pad) + except OverflowError: + raise StructError("total struct size too long") class PackFormatIterator(FormatIterator): Modified: pypy/dist/pypy/module/struct/interp_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/interp_struct.py (original) +++ pypy/dist/pypy/module/struct/interp_struct.py Thu Sep 27 14:14:26 2007 @@ -6,19 +6,12 @@ from pypy.module.struct.formatiterator import UnpackFormatIterator -def overflow(space): - return OperationError(space.w_OverflowError, - space.wrap("struct format too large")) - - def calcsize(space, format): fmtiter = CalcSizeFormatIterator() try: fmtiter.interpret(format) except StructError, e: raise e.at_applevel(space) - except OverflowError: - raise overflow(space) return space.wrap(fmtiter.totalsize) calcsize.unwrap_spec = [ObjSpace, str] @@ -29,8 +22,6 @@ fmtiter.interpret(format) except StructError, e: raise e.at_applevel(space) - except OverflowError: - raise overflow(space) result = ''.join(fmtiter.result) return space.wrap(result) pack.unwrap_spec = [ObjSpace, str, 'args_w'] @@ -42,7 +33,5 @@ fmtiter.interpret(format) except StructError, e: raise e.at_applevel(space) - except OverflowError: - raise overflow(space) return space.newtuple(fmtiter.result_w) unpack.unwrap_spec = [ObjSpace, str, str] From arigo at codespeak.net Thu Sep 27 14:17:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 14:17:54 +0200 (CEST) Subject: [pypy-svn] r46950 - pypy/dist/pypy/interpreter Message-ID: <20070927121754.9E0AA8142@code0.codespeak.net> Author: arigo Date: Thu Sep 27 14:17:54 2007 New Revision: 46950 Modified: pypy/dist/pypy/interpreter/argument.py Log: This assert helps the annotator figure out impossible cases. Motivated by "checkmodule.py struct". Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Thu Sep 27 14:17:54 2007 @@ -524,6 +524,7 @@ if self.w_stararg is None: # common case args_left = co_argcount - blindargs if args_left < 0: # check required by rpython + assert extravarargs is not None starargs_w = extravarargs if len(args_w): starargs_w.extend(args_w) From arigo at codespeak.net Thu Sep 27 14:44:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 14:44:08 +0200 (CEST) Subject: [pypy-svn] r46956 - pypy/dist/pypy/module/struct Message-ID: <20070927124408.5564A8170@code0.codespeak.net> Author: arigo Date: Thu Sep 27 14:44:08 2007 New Revision: 46956 Modified: pypy/dist/pypy/module/struct/standardfmttable.py Log: Translation fix. Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Thu Sep 27 14:44:08 2007 @@ -69,6 +69,8 @@ max = (2 ** (8*size-1)) - 1 if size <= native_int_size: accept_method = 'accept_int_arg' + min = int(min) + max = int(max) else: accept_method = 'accept_longlong_arg' min = r_longlong(min) From arigo at codespeak.net Thu Sep 27 15:18:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 15:18:26 +0200 (CEST) Subject: [pypy-svn] r46957 - pypy/dist/pypy/module/struct Message-ID: <20070927131826.65755815D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 15:18:24 2007 New Revision: 46957 Modified: pypy/dist/pypy/module/struct/formatiterator.py Log: Each of the three versions of interpret() is quite big. We can seriously reduce the size of the one used by calcsize() with what I think is a reasonable performance overhead. Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Thu Sep 27 15:18:24 2007 @@ -16,6 +16,7 @@ just computing the size. """ _mixin_ = True + _operate_is_specialized_ = False def interpret(self, fmt): # decode the byte order, size and alignment based on the 1st char @@ -66,12 +67,17 @@ for fmtdesc in table: if c == fmtdesc.fmtchar: - if fmtdesc.alignment > 1: - self.align(fmtdesc.mask) - self.operate(fmtdesc, repetitions) + if self._operate_is_specialized_: + if fmtdesc.alignment > 1: + self.align(fmtdesc.mask) + self.operate(fmtdesc, repetitions) break else: raise StructError("bad char in struct format") + if not self._operate_is_specialized_: + if fmtdesc.alignment > 1: + self.align(fmtdesc.mask) + self.operate(fmtdesc, repetitions) self.finished() def finished(self): @@ -83,15 +89,10 @@ def operate(self, fmtdesc, repetitions): try: - if fmtdesc.size == 1: - # skip the overflow-checked multiplication by 1 - size = repetitions - else: - size = ovfcheck(fmtdesc.size * repetitions) + size = ovfcheck(fmtdesc.size * repetitions) self.totalsize = ovfcheck(self.totalsize + size) except OverflowError: raise StructError("total struct size too long") - operate._annspecialcase_ = 'specialize:arg(1)' def align(self, mask): pad = (-self.totalsize) & mask @@ -116,6 +117,7 @@ for i in range(repetitions): fmtdesc.pack(self) operate._annspecialcase_ = 'specialize:arg(1)' + _operate_is_specialized_ = True def align(self, mask): pad = (-len(self.result)) & mask @@ -174,6 +176,7 @@ for i in range(repetitions): fmtdesc.unpack(self) operate._annspecialcase_ = 'specialize:arg(1)' + _operate_is_specialized_ = True def align(self, mask): self.inputpos = (self.inputpos + mask) & ~mask From arigo at codespeak.net Thu Sep 27 15:31:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 15:31:52 +0200 (CEST) Subject: [pypy-svn] r46959 - pypy/dist/pypy/module/zlib Message-ID: <20070927133152.BC44B8141@code0.codespeak.net> Author: arigo Date: Thu Sep 27 15:31:52 2007 New Revision: 46959 Modified: pypy/dist/pypy/module/zlib/interp_zlib.py Log: It seems that I didn't understand the purpose of the flush() method of decompression objects after all. Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Thu Sep 27 15:31:52 2007 @@ -218,11 +218,6 @@ decompress(data[, max_length]) -- Return a string containing the decompressed version of the data. - After calling this function, some of the input data may still be stored - in internal buffers for later processing. - - Call the flush() method to clear these buffers. - If the max_length parameter is specified then the return value will be no longer than max_length. Unconsumed input data will be stored in the unconsumed_tail attribute. @@ -240,17 +235,16 @@ def flush(self, length=0): """ - flush( [length] ) -- Return a string containing any remaining - decompressed data. length, if given, is the initial size of the output - buffer. - - The decompressor object can no longer be used after this call. + flush( [length] ) -- This is kept for backward compatibility, + because each call to decompress() immediately returns as much + data as possible. """ - try: - result = rzlib.decompress(self.stream, '', rzlib.Z_FINISH) - except rzlib.RZlibError, e: - raise zlib_error(self.space, e.msg) - return self.space.wrap(result) + # We could call rzlib.decompress(self.stream, '', rzlib.Z_FINISH) + # which would complain if the input stream so far is not complete; + # however CPython's zlib module does not behave like that. + # I could not figure out a case in which flush() in CPython + # simply returns an empty string without complaining. + return self.space.wrap("") flush.unwrap_spec = ['self', int] From arigo at codespeak.net Thu Sep 27 15:51:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 15:51:10 +0200 (CEST) Subject: [pypy-svn] r46960 - in pypy/dist/pypy: config doc/config lib Message-ID: <20070927135110.22087815A@code0.codespeak.net> Author: arigo Date: Thu Sep 27 15:51:08 2007 New Revision: 46960 Added: pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt (contents, props changed) Modified: pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/lib/struct.py Log: Add 'struct' to --allworkingmodules. Document its existence. Document the reason for keeping pypy/lib/struct.py. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Thu Sep 27 15:51:08 2007 @@ -24,6 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", + "struct", ] )) Added: pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt Thu Sep 27 15:51:08 2007 @@ -0,0 +1,4 @@ +Use the built-in 'struct' module. +This module is expected to be working and is included by default. +There is also a pure Python version in pypy/lib, but it is several +orders of magnitude slower. Modified: pypy/dist/pypy/lib/struct.py ============================================================================== --- pypy/dist/pypy/lib/struct.py (original) +++ pypy/dist/pypy/lib/struct.py Thu Sep 27 15:51:08 2007 @@ -1,3 +1,10 @@ +# +# This module is a pure Python version of pypy.module.struct. +# It is only imported if the vastly faster pypy.module.struct is not +# compiled in. For now we keep this version for reference and +# because pypy.module.struct is not ootype-backend-friendly yet. +# + """Functions to convert between Python values and C structs. Python strings are used to hold the data representing the C struct and also as format strings to describe the layout of data in the C struct. From arigo at codespeak.net Thu Sep 27 16:12:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 16:12:08 +0200 (CEST) Subject: [pypy-svn] r46961 - pypy/dist/pypy/rpython/test Message-ID: <20070927141208.0DC188141@code0.codespeak.net> Author: arigo Date: Thu Sep 27 16:12:07 2007 New Revision: 46961 Modified: pypy/dist/pypy/rpython/test/test_rint.py Log: Enhance this test to show a case where we need a Python long object in order to express a constant r_uint(). Modified: pypy/dist/pypy/rpython/test/test_rint.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rint.py (original) +++ pypy/dist/pypy/rpython/test/test_rint.py Thu Sep 27 16:12:07 2007 @@ -100,9 +100,10 @@ assert res == '-' + oct(sys.maxint+1).replace('L', '').replace('l', '') def test_unsigned(self): + bigvalue = sys.maxint + 17 def dummy(i): i = r_uint(i) - j = r_uint(12) + j = r_uint(bigvalue) return i < j res = self.interpret(dummy,[0]) From arigo at codespeak.net Thu Sep 27 16:58:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 16:58:03 +0200 (CEST) Subject: [pypy-svn] r46963 - in pypy/dist/pypy/rlib: . test Message-ID: <20070927145803.942828148@code0.codespeak.net> Author: arigo Date: Thu Sep 27 16:58:03 2007 New Revision: 46963 Added: pypy/dist/pypy/rlib/rmd5.py - copied, changed from r46960, pypy/dist/pypy/lib/md5.py pypy/dist/pypy/rlib/test/test_rmd5.py (contents, props changed) Log: Porting the md5 module to RPython... Added: pypy/dist/pypy/rlib/test/test_rmd5.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/test/test_rmd5.py Thu Sep 27 16:58:03 2007 @@ -0,0 +1,33 @@ +from pypy.rlib import rmd5 + + +def test_digest_size(): + assert rmd5.digest_size == 16 + + +def test_cases(): + """ + Feed example strings into a md5 object and check the digest and + hexdigest. + """ + cases = ( + ("", + "d41d8cd98f00b204e9800998ecf8427e"), + ("a", + "0cc175b9c0f1b6a831c399e269772661"), + ("abc", + "900150983cd24fb0d6963f7d28e17f72"), + ("message digest", + "f96b697d7cb7938d525a2f31aaf161d0"), + ("abcdefghijklmnopqrstuvwxyz", + "c3fcd3d76192e4007dfb496cca67e13b"), + ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f"), + ("1234567890"*8, + "57edf4a22be3c955ac49da2e2107b67a"), + ) + for input, expected in cases: + d = rmd5.RMD5() + d.update(input) + assert d.hexdigest() == expected + assert d.digest() == expected.decode('hex') From arigo at codespeak.net Thu Sep 27 17:06:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 17:06:24 +0200 (CEST) Subject: [pypy-svn] r46964 - in pypy/dist/pypy/rlib: . test Message-ID: <20070927150624.120FC815D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 17:06:23 2007 New Revision: 46964 Modified: pypy/dist/pypy/rlib/rmd5.py pypy/dist/pypy/rlib/test/test_rmd5.py Log: Port the tests from lib/test/test_md5.py. Fix a bug. Modified: pypy/dist/pypy/rlib/rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/rmd5.py (original) +++ pypy/dist/pypy/rlib/rmd5.py Thu Sep 27 17:06:23 2007 @@ -65,10 +65,10 @@ hx[(d>>20)&0xF], hx[(d>>16)&0xF], hx[(d>>28)&0xF], hx[(d>>24)&0xF], ]) -def _string2uintlist(s): +def _string2uintlist(s, start=0, count=16): result = [] - for i in range(len(s) // 4): - p = i * 4 + for i in range(count): + p = start + i * 4 x = r_uint(ord(s[p])) x |= r_uint(ord(s[p+1])) << 8 x |= r_uint(ord(s[p+2])) << 16 @@ -260,7 +260,7 @@ # Append length (before padding). assert len(self.input) == 56 - bits = _string2uintlist(self.input) + bits = _string2uintlist(self.input, 0, 56 // 4) length_in_bits = count << 3 bits.append(r_uint(length_in_bits)) bits.append(r_uint(length_in_bits >> 32)) Modified: pypy/dist/pypy/rlib/test/test_rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmd5.py (original) +++ pypy/dist/pypy/rlib/test/test_rmd5.py Thu Sep 27 17:06:23 2007 @@ -1,3 +1,4 @@ +import md5 # for comparison from pypy.rlib import rmd5 @@ -31,3 +32,109 @@ d.update(input) assert d.hexdigest() == expected assert d.digest() == expected.decode('hex') + + +def test_more(): + "Test cases without known digest result." + + cases = ( + "123", + "1234", + "12345", + "123456", + "1234567", + "12345678", + "123456789 123456789 123456789 ", + "123456789 123456789 ", + "123456789 123456789 1", + "123456789 123456789 12", + "123456789 123456789 123", + "123456789 123456789 1234", + "123456789 123456789 123456789 1", + "123456789 123456789 123456789 12", + "123456789 123456789 123456789 123", + "123456789 123456789 123456789 1234", + "123456789 123456789 123456789 12345", + "123456789 123456789 123456789 123456", + "123456789 123456789 123456789 1234567", + "123456789 123456789 123456789 12345678", + ) + + for input in cases: + d = rmd5.RMD5(input) + assert d.hexdigest() == md5.md5(input).hexdigest() + assert d.digest() == md5.md5(input).digest() + + +def test_long(): + "Test cases with long messages (can take a while)." + + cases = ( + 2**10*'a', + 2**10*'abcd', + ##2**20*'a', ## 1 MB, takes about 160 sec. on a 233 Mhz Pentium. + ) + + for input in cases: + d = rmd5.RMD5(input) + assert d.hexdigest() == md5.md5(input).hexdigest() + assert d.digest() == md5.md5(input).digest() + + +def test_updating_many_times(): + "Test with an increasingly growing message." + + d1 = rmd5.RMD5() + d2 = md5.md5() + for i in range(300): + d1.update(chr(i & 0xFF)) + d2.update(chr(i & 0xFF)) + assert d1.digest() == d2.digest() + + +def test_copy(): + "Test updating cloned objects." + + cases = ( + "123", + "1234", + "12345", + "123456", + "1234567", + "12345678", + "123456789 123456789 123456789 ", + "123456789 123456789 ", + "123456789 123456789 1", + "123456789 123456789 12", + "123456789 123456789 123", + "123456789 123456789 1234", + "123456789 123456789 123456789 1", + "123456789 123456789 123456789 12", + "123456789 123456789 123456789 123", + "123456789 123456789 123456789 1234", + "123456789 123456789 123456789 12345", + "123456789 123456789 123456789 123456", + "123456789 123456789 123456789 1234567", + "123456789 123456789 123456789 12345678", + ) + + # Load both with same prefix. + prefix1 = 2**10 * 'a' + + m1 = md5.md5() + m1.update(prefix1) + + m2 = rmd5.RMD5() + m2.update(prefix1) + + # Update and compare... + for message in cases: + m1c = m1.copy() + m1c.update(message) + d1 = m1c.hexdigest() + + m2c = m2.copy() + m2c.update(message) + d2 = m2c.hexdigest() + + assert d1 == d2 From arigo at codespeak.net Thu Sep 27 17:18:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 17:18:48 +0200 (CEST) Subject: [pypy-svn] r46965 - in pypy/dist/pypy/module/md5: . test Message-ID: <20070927151848.F06B5818B@code0.codespeak.net> Author: arigo Date: Thu Sep 27 17:18:47 2007 New Revision: 46965 Added: pypy/dist/pypy/module/md5/ (props changed) pypy/dist/pypy/module/md5/__init__.py (contents, props changed) pypy/dist/pypy/module/md5/interp_md5.py (contents, props changed) pypy/dist/pypy/module/md5/test/ (props changed) pypy/dist/pypy/module/md5/test/test_md5.py (contents, props changed) Log: An interp-level md5 module for PyPy. Added: pypy/dist/pypy/module/md5/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/md5/__init__.py Thu Sep 27 17:18:47 2007 @@ -0,0 +1,29 @@ + +""" +Mixed-module definition for the md5 module. +Note that there is also a pure Python implementation in pypy/lib/md5.py; +the present mixed-module version of md5 takes precedence if it is enabled. +""" + +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + """\ +This module implements the interface to RSA's MD5 message digest +algorithm (see also Internet RFC 1321). Its use is quite +straightforward: use new() to create an md5 object. You can now feed +this object with arbitrary strings using the update() method, and at any +point you can ask it for the digest (a strong kind of 128-bit checksum, +a.k.a. ``fingerprint'') of the concatenation of the strings fed to it so +far using the digest() method.""" + + interpleveldefs = { + 'md5': 'interp_md5.W_MD5', + 'new': 'interp_md5.W_MD5', + 'MD5Type': 'interp_md5.W_MD5', + 'digest_size': 'space.wrap(16)', + } + + appleveldefs = { + } Added: pypy/dist/pypy/module/md5/interp_md5.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/md5/interp_md5.py Thu Sep 27 17:18:47 2007 @@ -0,0 +1,47 @@ +from pypy.rlib import rmd5 +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root + + +class W_MD5(Wrappable, rmd5.RMD5): + """ + A subclass of RMD5 that can be exposed to app-level. + """ + + def __init__(self, space, initialdata=''): + self.space = space + self._init() + self.update(initialdata) + + def digest_w(self): + return self.space.wrap(self.digest()) + + def hexdigest_w(self): + return self.space.wrap(self.hexdigest()) + + def copy_w(self): + return self.space.wrap(self.copy()) + + +def W_MD5___new__(space, w_subtype, initialdata=''): + """ + Create a new md5 object and call its initializer. + """ + w_md5 = space.allocate_instance(W_MD5, w_subtype) + md5 = space.interp_w(W_MD5, w_md5) + W_MD5.__init__(md5, space, initialdata) + return w_md5 +W_MD5___new__.unwrap_spec = [ObjSpace, W_Root, str] + + +W_MD5.typedef = TypeDef( + 'MD5Type', + __new__ = interp2app(W_MD5___new__), + update = interp2app(W_MD5.update, unwrap_spec=['self', str]), + digest = interp2app(W_MD5.digest_w), + hexdigest = interp2app(W_MD5.hexdigest_w), + copy = interp2app(W_MD5.copy_w), + __doc__ = """md5(arg) -> return new md5 object. + +If arg is present, the method call update(arg) is made.""") Added: pypy/dist/pypy/module/md5/test/test_md5.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/md5/test/test_md5.py Thu Sep 27 17:18:47 2007 @@ -0,0 +1,80 @@ +""" +Tests for the md5 module implemented at interp-level in pypy/module/md5. +""" + +import py +from pypy.conftest import gettestobjspace + + +class AppTestMD5(object): + + def setup_class(cls): + """ + Create a space with the md5 module and import it for use by the + tests. + """ + cls.space = gettestobjspace(usemodules=['md5']) + cls.w_md5 = cls.space.appexec([], """(): + import md5 + return md5 + """) + + + def test_digest_size(self): + """ + md5.digest_size should be 16. + """ + assert self.md5.digest_size == 16 + + + def test_MD5Type(self): + """ + Test the two ways to construct an md5 object. + """ + md5 = self.md5 + d = md5.md5() + assert isinstance(d, md5.MD5Type) + d = md5.new() + assert isinstance(d, md5.MD5Type) + + + def test_md5object(self): + """ + Feed example strings into a md5 object and check the digest and + hexdigest. + """ + md5 = self.md5 + cases = ( + ("", + "d41d8cd98f00b204e9800998ecf8427e"), + ("a", + "0cc175b9c0f1b6a831c399e269772661"), + ("abc", + "900150983cd24fb0d6963f7d28e17f72"), + ("message digest", + "f96b697d7cb7938d525a2f31aaf161d0"), + ("abcdefghijklmnopqrstuvwxyz", + "c3fcd3d76192e4007dfb496cca67e13b"), + ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f"), + ("1234567890"*8, + "57edf4a22be3c955ac49da2e2107b67a"), + ) + for input, expected in cases: + d = md5.new(input) + assert d.hexdigest() == expected + assert d.digest() == expected.decode('hex') + + + def test_copy(self): + """ + Test the copy() method. + """ + md5 = self.md5 + d1 = md5.md5() + d1.update("abcde") + d2 = d1.copy() + d2.update("fgh") + d1.update("jkl") + assert d1.hexdigest() == 'e570e7110ecef72fcb772a9c05d03373' + assert d2.hexdigest() == 'e8dc4081b13434b45189a720b77b6818' From cfbolz at codespeak.net Thu Sep 27 17:24:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 27 Sep 2007 17:24:36 +0200 (CEST) Subject: [pypy-svn] r46966 - in pypy/dist/pypy/rlib: . test Message-ID: <20070927152436.0DCEE8148@code0.codespeak.net> Author: cfbolz Date: Thu Sep 27 17:24:35 2007 New Revision: 46966 Modified: pypy/dist/pypy/rlib/rmd5.py pypy/dist/pypy/rlib/test/test_rmd5.py Log: a random string sort of test for rmd5. add an encoding declaration in rmd5.py for Jacobs surname Modified: pypy/dist/pypy/rlib/rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/rmd5.py (original) +++ pypy/dist/pypy/rlib/rmd5.py Thu Sep 27 17:24:35 2007 @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ RPython implementation of MD5 checksums. Modified: pypy/dist/pypy/rlib/test/test_rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rmd5.py (original) +++ pypy/dist/pypy/rlib/test/test_rmd5.py Thu Sep 27 17:24:35 2007 @@ -138,3 +138,15 @@ d2 = m2c.hexdigest() assert d1 == d2 + +def test_random(): + import random, md5 + for i in range(20): + input = ''.join([chr(random.randrange(256)) + for i in range(random.randrange(1000))]) + m1 = rmd5.RMD5() + m1.update(input) + m2 = md5.new() + m2.update(input) + assert m2.hexdigest() == m1.hexdigest() + From arigo at codespeak.net Thu Sep 27 17:25:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 17:25:39 +0200 (CEST) Subject: [pypy-svn] r46967 - pypy/dist/pypy/rlib Message-ID: <20070927152539.EF342818B@code0.codespeak.net> Author: arigo Date: Thu Sep 27 17:25:39 2007 New Revision: 46967 Modified: pypy/dist/pypy/rlib/rmd5.py Log: Forgot to check this in. Modified: pypy/dist/pypy/rlib/rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/rmd5.py (original) +++ pypy/dist/pypy/rlib/rmd5.py Thu Sep 27 17:25:39 2007 @@ -124,6 +124,13 @@ _mixin_ = True # for interp_md5.py def __init__(self, initialdata=''): + self._init() + self.update(initialdata) + + + def _init(self): + """Set this object to an initial empty state. + """ self.count = r_ulonglong(0) # total number of bytes self.input = "" # pending unprocessed data, < 64 bytes @@ -133,8 +140,6 @@ self.C = r_uint(0x98badcfeL) self.D = r_uint(0x10325476L) - self.update(initialdata) - def _transform(self, inp): """Basic MD5 step transforming the digest based on the input. From arigo at codespeak.net Thu Sep 27 17:44:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 17:44:58 +0200 (CEST) Subject: [pypy-svn] r46968 - pypy/dist/pypy/module/zlib Message-ID: <20070927154458.633568171@code0.codespeak.net> Author: arigo Date: Thu Sep 27 17:44:58 2007 New Revision: 46968 Modified: pypy/dist/pypy/module/zlib/interp_zlib.py Log: Typo (thanks xorAxAx) Modified: pypy/dist/pypy/module/zlib/interp_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/interp_zlib.py (original) +++ pypy/dist/pypy/module/zlib/interp_zlib.py Thu Sep 27 17:44:58 2007 @@ -243,7 +243,7 @@ # which would complain if the input stream so far is not complete; # however CPython's zlib module does not behave like that. # I could not figure out a case in which flush() in CPython - # simply returns an empty string without complaining. + # doesn't simply return an empty string without complaining. return self.space.wrap("") flush.unwrap_spec = ['self', int] From arigo at codespeak.net Thu Sep 27 18:51:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 18:51:45 +0200 (CEST) Subject: [pypy-svn] r46969 - pypy/dist/pypy/module/md5 Message-ID: <20070927165145.43C76813D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 18:51:39 2007 New Revision: 46969 Modified: pypy/dist/pypy/module/md5/interp_md5.py Log: Checking this in, but I'm not getting why annotation fails... Modified: pypy/dist/pypy/module/md5/interp_md5.py ============================================================================== --- pypy/dist/pypy/module/md5/interp_md5.py (original) +++ pypy/dist/pypy/module/md5/interp_md5.py Thu Sep 27 18:51:39 2007 @@ -14,6 +14,9 @@ self._init() self.update(initialdata) + def update_w(self, string): + self.update(string) + def digest_w(self): return self.space.wrap(self.digest()) @@ -32,16 +35,15 @@ md5 = space.interp_w(W_MD5, w_md5) W_MD5.__init__(md5, space, initialdata) return w_md5 -W_MD5___new__.unwrap_spec = [ObjSpace, W_Root, str] W_MD5.typedef = TypeDef( 'MD5Type', - __new__ = interp2app(W_MD5___new__), - update = interp2app(W_MD5.update, unwrap_spec=['self', str]), - digest = interp2app(W_MD5.digest_w), - hexdigest = interp2app(W_MD5.hexdigest_w), - copy = interp2app(W_MD5.copy_w), + __new__ = interp2app(W_MD5___new__, unwrap_spec=[ObjSpace, W_Root, str]), + update = interp2app(W_MD5.update_w, unwrap_spec=['self', str]), + digest = interp2app(W_MD5.digest_w, unwrap_spec=['self']), + hexdigest = interp2app(W_MD5.hexdigest_w, unwrap_spec=['self']), + copy = interp2app(W_MD5.copy_w, unwrap_spec=['self']), __doc__ = """md5(arg) -> return new md5 object. If arg is present, the method call update(arg) is made.""") From arigo at codespeak.net Thu Sep 27 18:57:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 18:57:19 +0200 (CEST) Subject: [pypy-svn] r46970 - in pypy/dist/pypy: module/md5 rlib Message-ID: <20070927165719.67ECF813D@code0.codespeak.net> Author: arigo Date: Thu Sep 27 18:57:19 2007 New Revision: 46970 Modified: pypy/dist/pypy/module/md5/interp_md5.py pypy/dist/pypy/rlib/rmd5.py Log: fix: RMD5.copy() returned a RMD5 instance instead of a W_MD5 one. Modified: pypy/dist/pypy/module/md5/interp_md5.py ============================================================================== --- pypy/dist/pypy/module/md5/interp_md5.py (original) +++ pypy/dist/pypy/module/md5/interp_md5.py Thu Sep 27 18:57:19 2007 @@ -9,10 +9,9 @@ A subclass of RMD5 that can be exposed to app-level. """ - def __init__(self, space, initialdata=''): + def __init__(self, space): self.space = space self._init() - self.update(initialdata) def update_w(self, string): self.update(string) @@ -24,7 +23,9 @@ return self.space.wrap(self.hexdigest()) def copy_w(self): - return self.space.wrap(self.copy()) + clone = W_MD5(self.space) + clone._copyfrom(self) + return self.space.wrap(clone) def W_MD5___new__(space, w_subtype, initialdata=''): @@ -33,7 +34,8 @@ """ w_md5 = space.allocate_instance(W_MD5, w_subtype) md5 = space.interp_w(W_MD5, w_md5) - W_MD5.__init__(md5, space, initialdata) + W_MD5.__init__(md5, space) + md5.update(initialdata) return w_md5 Modified: pypy/dist/pypy/rlib/rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/rmd5.py (original) +++ pypy/dist/pypy/rlib/rmd5.py Thu Sep 27 18:57:19 2007 @@ -352,14 +352,18 @@ a common initial substring. """ clone = RMD5() - clone.count = self.count - clone.input = self.input - clone.A = self.A - clone.B = self.B - clone.C = self.C - clone.D = self.D + clone._copyfrom(self) return clone + def _copyfrom(self, other): + """Copy all state from 'other' into 'self'. + """ + self.count = other.count + self.input = other.input + self.A = other.A + self.B = other.B + self.C = other.C + self.D = other.D # synonyms to build new RMD5 objects, for compatibility with the # CPython md5 module interface. From arigo at codespeak.net Thu Sep 27 19:45:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Sep 2007 19:45:34 +0200 (CEST) Subject: [pypy-svn] r46972 - pypy/dist/pypy/rlib Message-ID: <20070927174534.4EF73818F@code0.codespeak.net> Author: arigo Date: Thu Sep 27 19:45:33 2007 New Revision: 46972 Modified: pypy/dist/pypy/rlib/rmd5.py Log: Mumble picky rtyper mumble Modified: pypy/dist/pypy/rlib/rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/rmd5.py (original) +++ pypy/dist/pypy/rlib/rmd5.py Thu Sep 27 19:45:33 2007 @@ -309,6 +309,7 @@ self.count += leninBuf index = len(self.input) partLen = 64 - index + assert partLen > 0 if leninBuf >= partLen: self.input = self.input + inBuf[:partLen] From antocuni at codespeak.net Thu Sep 27 21:58:38 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 27 Sep 2007 21:58:38 +0200 (CEST) Subject: [pypy-svn] r46974 - pypy/dist/pypy/translator/cli/test Message-ID: <20070927195838.1F3968187@code0.codespeak.net> Author: antocuni Date: Thu Sep 27 21:58:37 2007 New Revision: 46974 Added: pypy/dist/pypy/translator/cli/test/test_weakref.py (contents, props changed) Log: I previously forgot to checkin this! Added: pypy/dist/pypy/translator/cli/test/test_weakref.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/cli/test/test_weakref.py Thu Sep 27 21:58:37 2007 @@ -0,0 +1,6 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rweakref import BaseTestRweakref + +class TestCliWeakRef(CliTest, BaseTestRweakref): + pass From niko at codespeak.net Thu Sep 27 22:17:18 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 27 Sep 2007 22:17:18 +0200 (CEST) Subject: [pypy-svn] r46975 - pypy/dist/pypy/translator/jvm Message-ID: <20070927201718.85F2B8193@code0.codespeak.net> Author: niko Date: Thu Sep 27 22:17:16 2007 New Revision: 46975 Modified: pypy/dist/pypy/translator/jvm/genjvm.py Log: modify genjvm so that it auto-detects the .java files and compiles them appropriately, rather than hardcoding the class names in the python code. DRY, and all that. Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Thu Sep 27 22:17:16 2007 @@ -107,8 +107,8 @@ raise JvmSubprogramError(res, args, stdout, stderr) return stdout, stderr - def _compile_helper(self, clsnms): - # HACK: compile the Java helper class. Should eventually + def _compile_helper(self): + # HACK: compile the Java helper classes. Should eventually # use rte.py thisdir = py.magic.autopath().dirpath() rootdir = thisdir.join('src') @@ -117,7 +117,7 @@ classfiles = srcdir.listdir('*.class') recompile = True - if classfiles: + if len(classfiles) == len(javafiles): last_modified_java = max([java.mtime() for java in javafiles]) last_modified_class = max([cls.mtime() for cls in classfiles]) if last_modified_java < last_modified_class: @@ -125,7 +125,7 @@ if recompile: log.red('Compiling java classes') - javasrcs = [str(srcdir.join(clsnm + '.java')) for clsnm in clsnms] + javasrcs = [str(jf) for jf in javafiles] self._invoke([getoption('javac'), '-nowarn', '-d', str(rootdir)] + javasrcs, True) # copy .class files to classdir @@ -166,20 +166,7 @@ print "... completed!" self.compiled = True - self._compile_helper(('Callback', - 'CustomDict', - 'DictItemsIterator', - 'Equals', - 'Filter', - 'FilterIterator', - 'FilterSet', - 'HashCode', - 'Interlink', - 'Constants', - 'StatResult', - 'PyPy', - 'll_os', - )) + self._compile_helper() def _make_str(self, a): if isinstance(a, ootype._string): From niko at codespeak.net Thu Sep 27 22:18:41 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 27 Sep 2007 22:18:41 +0200 (CEST) Subject: [pypy-svn] r46976 - in pypy/dist/pypy/translator/jvm: . src/pypy Message-ID: <20070927201841.7FBFC819B@code0.codespeak.net> Author: niko Date: Thu Sep 27 22:18:41 2007 New Revision: 46976 Added: pypy/dist/pypy/translator/jvm/src/pypy/PyPyWeakRef.java Modified: pypy/dist/pypy/translator/jvm/builtin.py pypy/dist/pypy/translator/jvm/constant.py pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/metavm.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/typesystem.py Log: implement new-style weakrefs in the jvm Modified: pypy/dist/pypy/translator/jvm/builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/builtin.py (original) +++ pypy/dist/pypy/translator/jvm/builtin.py Thu Sep 27 22:18:41 2007 @@ -52,7 +52,7 @@ ARGS, RESULT = self.gen.erased_types(methodnm) jargtypes = [self.db.lltype_to_cts(P) for P in ARGS] jrettype = self.db.lltype_to_cts(RESULT) - + if self.OOTYPE.__class__ in bridged_objects: # Bridged objects are ones where we have written a java class # that has methods with the correct names and types already @@ -71,6 +71,7 @@ bridged_objects = ( ootype.DictItemsIterator, + ootype.WeakReference.__class__ ) built_in_methods = { Modified: pypy/dist/pypy/translator/jvm/constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/constant.py (original) +++ pypy/dist/pypy/translator/jvm/constant.py Thu Sep 27 22:18:41 2007 @@ -7,7 +7,7 @@ StaticMethodConst, CustomDictConst, WeakRefConst, push_constant, \ MAX_CONST_PER_STEP from pypy.translator.jvm.typesystem import \ - jObject, jVoid, jWeakRef, JvmClassType + jObject, jVoid, jPyPyWeakRef, JvmClassType jPyPyConstantInit = JvmClassType('pypy.ConstantInit') jPyPyConstantInitMethod = Method.s(jPyPyConstantInit, 'init', [], jVoid) @@ -188,17 +188,16 @@ PRIORITY = 200 def jtype(self): - return jWeakRef + return jPyPyWeakRef - def create_pointer(self, gen): - gen.prepare_cast_ptr_to_weak_address() + def create_pointer(self, gen): if not self.value: TYPE = ootype.ROOT gen.push_null(TYPE) else: TYPE = self.value._TYPE push_constant(self.db, self.value._TYPE, self.value, gen) - gen.finalize_cast_ptr_to_weak_address(TYPE) + gen.create_weakref(TYPE) def initialize_data(self, constgen, gen): gen.pop(ootype.ROOT) Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Thu Sep 27 22:18:41 2007 @@ -460,7 +460,6 @@ ootype.UniChar: jvmtype.jChar, ootype.Class: jvmtype.jClass, ootype.ROOT: jvmtype.jObject, # treat like a scalar - #WeakGcAddress: jvmtype.jWeakRef, } # Dictionary for non-scalar types; in this case, if we see the key, we @@ -472,6 +471,7 @@ ootype.Dict: jvmtype.jHashMap, ootype.DictItemsIterator:jvmtype.jPyPyDictItemsIterator, ootype.CustomDict: jvmtype.jPyPyCustomDict, + ootype.WeakReference: jvmtype.jPyPyWeakRef, ll_os.STAT_RESULT: jvmtype.jPyPyStatResult, } Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Thu Sep 27 22:18:41 2007 @@ -12,7 +12,7 @@ jObject, jByteArray, jPyPyExcWrap, jIntegerClass, jLongClass, \ jDoubleClass, jCharClass, jStringBuilder, JvmScalarType, jArrayList, \ jObjectArray, jPyPyInterlink, jPyPyCustomDict, jPyPyEquals, \ - jPyPyHashCode, jMap, jWeakRef, jSystem, jll_os + jPyPyHashCode, jMap, jPyPyWeakRef, jSystem, jll_os # ___________________________________________________________________________ # Miscellaneous helper functions @@ -429,6 +429,8 @@ CLASSGETNAME = Method.v(jClass, 'getName', (), jString) CUSTOMDICTMAKE = Method.s(jPyPyCustomDict, 'make', (jPyPyEquals, jPyPyHashCode), jPyPyCustomDict) +PYPYWEAKREFCREATE = Method.s(jPyPyWeakRef, 'create', (jObject,), jPyPyWeakRef) +PYPYWEAKREFGET = Method.v(jPyPyWeakRef, 'll_get', (), jObject) # ___________________________________________________________________________ # Fields @@ -1137,17 +1139,8 @@ else: # Big hack to avoid exponential notation: self.emit(LDC2, "%22.22f" % value) - - def prepare_cast_ptr_to_weak_address(self): - """ - To cast a pointer to a weak ref is a 2-step process. - First, invoke this routine. Then, invoke what is needed - to push the value, then invoke finalize_cast_ptr_to_weak_address - """ - self.emit(NEW, jWeakRef) - self.emit(DUP) - def finalize_cast_ptr_to_weak_address(self, OOTYPE): + def create_weakref(self, OOTYPE): """ After prepare_cast_ptr_to_weak_address has been called, and the ptr to cast has been pushed, you can invoke this routine. @@ -1155,17 +1148,15 @@ The result will be that at the top of the stack is a weak reference. """ self.prepare_generic_argument(OOTYPE) - ctor = Method.c(jWeakRef, (jObject,)) - self.emit(ctor) - - def cast_weak_address_to_ptr(self, OOTYPE): + self.emit(PYPYWEAKREFCREATE) + + def deref_weakref(self, OOTYPE): """ If a weak ref is at the top of the stack, yields the object that this weak ref is a pointer to. OOTYPE is the kind of object you had a weak reference to. """ - get_mthd = Method.v(jWeakRef, 'get', (), jObject) - self.emit(get_mthd) + self.emit(PYPYWEAKREFGET) self.prepare_generic_result(OOTYPE) # __________________________________________________________________ Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Thu Sep 27 22:18:41 2007 @@ -126,20 +126,20 @@ generator.emit(jvmgen.CUSTOMDICTMAKE) NewCustomDict = _NewCustomDict() -#XXX adapt to new way of things +#XXX These classes have been adapted to the new +#XXX WeakRef methods, but don't appear to be needed. #class _CastPtrToWeakAddress(MicroInstruction): # def render(self, generator, op): # arg = op.args[0] -# generator.prepare_cast_ptr_to_weak_address() # generator.load(arg) -# generator.finalize_cast_ptr_to_weak_address(arg.concretetype) +# generator.create_weakref(arg.concretetype) # generator.store(op.result) #CastPtrToWeakAddress = _CastPtrToWeakAddress() #class _CastWeakAddressToPtr(MicroInstruction): # def render(self, generator, op): # RESULTTYPE = op.result.concretetype -# generator.cast_weak_address_to_ptr(RESULTTYPE) +# generator.deref_weakref(RESULTTYPE) #CastWeakAddressToPtr = _CastWeakAddressToPtr() Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Thu Sep 27 22:18:41 2007 @@ -65,6 +65,7 @@ 'direct_call': [Call, StoreResult], 'indirect_call': [PushAllArgs, IndirectCall, StoreResult], + #'cast_ptr_to_weakadr': CastPtrToWeakAddress 'gc__collect': jvmgen.SYSTEMGC, 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, Added: pypy/dist/pypy/translator/jvm/src/pypy/PyPyWeakRef.java ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPyWeakRef.java Thu Sep 27 22:18:41 2007 @@ -0,0 +1,23 @@ +package pypy; + +import java.lang.ref.WeakReference; + +public final class PyPyWeakRef { + WeakReference wref; + + public static PyPyWeakRef create(Object obj) { + PyPyWeakRef res = new PyPyWeakRef(); + res.ll_set(obj); + return res; + } + + public void ll_set(Object obj) + { + this.wref = new WeakReference(obj); + } + + public Object ll_deref() + { + return this.wref.get(); + } +} Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Thu Sep 27 22:18:41 2007 @@ -20,8 +20,8 @@ Python Unicode strings, on the other hand, map directly to Java Strings. -WeakRefs can hopefully map to Java Weak References in a straight -forward fashion. +WeakRefs are mapped to a thin wrapper class, PyPyWeakRef, to allow for +mutation of the object being referenced (the ll_set method). Collections can hopefully map to Java collections instances. Note that JVM does not have an idea of generic typing at its lowest level @@ -170,7 +170,6 @@ jMath = JvmClassType('java.lang.Math') jList = JvmInterfaceType('java.util.List') jArrayList = JvmClassType('java.util.ArrayList') -jWeakRef = JvmClassType('java.lang.ref.WeakReference') jPyPy = JvmClassType('pypy.PyPy') jPyPyExcWrap = JvmClassType('pypy.ExceptionWrapper') jPyPyMain = JvmClassType('pypy.Main') @@ -178,6 +177,7 @@ jPyPyInterlink = JvmClassType('pypy.Interlink') jPyPyCustomDict = JvmClassType('pypy.CustomDict') jPyPyStatResult = JvmClassType('pypy.StatResult') +jPyPyWeakRef = JvmClassType('pypy.PyPyWeakRef') jll_os = JvmClassType('pypy.ll_os') jArithmeticException = JvmClassType('java.lang.ArithmeticException', throwable=True) From niko at codespeak.net Thu Sep 27 22:19:03 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 27 Sep 2007 22:19:03 +0200 (CEST) Subject: [pypy-svn] r46977 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070927201903.5B6C5819E@code0.codespeak.net> Author: niko Date: Thu Sep 27 22:19:03 2007 New Revision: 46977 Added: pypy/dist/pypy/translator/jvm/test/test_weakref.py Log: add tests for jvm weakrefs Added: pypy/dist/pypy/translator/jvm/test/test_weakref.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_weakref.py Thu Sep 27 22:19:03 2007 @@ -0,0 +1,6 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rweakref import BaseTestRweakref + +class TestJvmException(JvmTest, BaseTestRweakref): + pass \ No newline at end of file From niko at codespeak.net Fri Sep 28 07:14:10 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 28 Sep 2007 07:14:10 +0200 (CEST) Subject: [pypy-svn] r46986 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20070928051410.58CFC8173@code0.codespeak.net> Author: niko Date: Fri Sep 28 07:14:09 2007 New Revision: 46986 Added: pypy/dist/pypy/translator/jvm/test/test_rarithmetic.py Modified: pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/test/runtest.py Log: 1. implement uint_div, ullong_lshift 2. port some of rlib's test_rarithmetic to the jvm in a rather hacky way... ...also it doens't test ullong_lshift Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Fri Sep 28 07:14:09 2007 @@ -139,8 +139,8 @@ 'uint_add': jvmgen.IADD, 'uint_sub': jvmgen.ISUB, 'uint_mul': jvmgen.PYPYUINTMUL, -# 'uint_div': jvmgen.IDIV, # valid? -# 'uint_truediv': None, # TODO + 'uint_div': jvmgen.PYPYUINTDIV, + 'uint_truediv': None, # TODO 'uint_floordiv': jvmgen.PYPYUINTDIV, 'uint_mod': jvmgen.PYPYUINTMOD, 'uint_lt': 'u_less_than', @@ -211,14 +211,15 @@ 'ullong_truediv': None, # TODO 'ullong_floordiv': jvmgen.LDIV, # valid? 'ullong_mod': jvmgen.PYPYULONGMOD, - 'ullong_mod_zer': _check_zer(jvmgen.PYPYULONGMOD), - 'ullong_rshift': [PushAllArgs, jvmgen.L2I, jvmgen.LUSHR, StoreResult], 'ullong_lt': 'ulong_less_than', 'ullong_le': 'ulong_less_equals', 'ullong_eq': 'ulong_equals', 'ullong_ne': 'ulong_not_equals', 'ullong_gt': 'ulong_greater_than', 'ullong_ge': 'ulong_greater_equals', + 'ullong_lshift': [PushAllArgs, jvmgen.L2I, jvmgen.LSHL, StoreResult], + 'ullong_rshift': [PushAllArgs, jvmgen.L2I, jvmgen.LUSHR, StoreResult], + 'ullong_mod_zer': _check_zer(jvmgen.PYPYULONGMOD), # when casting from bool we want that every truth value is casted # to 1: we can't simply DoNothing, because the CLI stack could @@ -226,8 +227,7 @@ # trick. #THIS COMMENT NEEDS TO BE VALIDATED AND UPDATED 'cast_bool_to_int': DoNothing, 'cast_bool_to_uint': DoNothing, - 'cast_bool_to_float': jvmgen.PYPYBOOLTODOUBLE, #PAUL, This should be replaced with JASMIN code later for spee, This should be replaced with JASMIN code later for speedd - + 'cast_bool_to_float': jvmgen.PYPYBOOLTODOUBLE, #PAUL, inefficient 'cast_char_to_int': DoNothing, 'cast_unichar_to_int': DoNothing, 'cast_int_to_char': DoNothing, Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Fri Sep 28 07:14:09 2007 @@ -69,9 +69,10 @@ } public static int uint_mod(int x, int y) { - double dx = uint_to_double(x); - double modulo = Math.IEEEremainder(dx, y); - return (int)modulo; + long lx = uint_to_long(x); + long ly = uint_to_long(y); + long lr = lx % ly; + return long_to_uint(lr); } public static int uint_mul(int x, int y) @@ -87,6 +88,11 @@ long yy = uint_to_long(y); return long_to_uint(xx / yy); } + + public static long ulong_shl(long x, long y) { + int yi = (int)y; + return x << yi; + } public static long ulong_mod(long x, long y) { double dx = ulong_to_double(x); @@ -131,7 +137,6 @@ return (double)uint_to_long(value); } - // XXX: broken if the value is too large public static double ulong_to_double(long value) { if (value >= 0) @@ -140,7 +145,7 @@ return ULONG_MAX + value; } } - + public static int double_to_uint(double value) { if (value <= Integer.MAX_VALUE) return (int)value; Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Fri Sep 28 07:14:09 2007 @@ -76,6 +76,8 @@ res = StructTuple(res) # so tests can access tuple elements with .item0, .item1, etc. elif isinstance(res, list): res = OOList(res) + elif isinstance(res, ExceptionWrapper): + raise res return res class JvmTest(BaseRtypingTest, OORtypeMixin): @@ -84,7 +86,7 @@ self._ann = None self._jvm_src = None - def _compile(self, fn, args, ann=None, backendopt=False): + def compile(self, fn, args, ann=None, backendopt=False): if ann is None: ann = [lltype_to_annotation(typeOf(x)) for x in args] if self._func is fn and self._ann == ann: @@ -113,10 +115,8 @@ def interpret(self, fn, args, annotation=None): detect_missing_support_programs() try: - src = self._compile(fn, args, annotation) + src = self.compile(fn, args, annotation) res = src(*args) - if isinstance(res, ExceptionWrapper): - raise res return res except JvmError, e: e.pretty_print() Added: pypy/dist/pypy/translator/jvm/test/test_rarithmetic.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_rarithmetic.py Fri Sep 28 07:14:09 2007 @@ -0,0 +1,50 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.test.test_rarithmetic import Test_r_uint as BaseTest_r_uint +from pypy.rlib.test.test_rarithmetic import Test_r_int as BaseTest_r_int +from pypy.rlib.test.test_rarithmetic import test_ovfcheck as base_test_ovfcheck +from pypy.rlib import rarithmetic as ra + +class BaseAdaptedTest(JvmTest): + + def unary_test(self, f): + cache = {} + def new_func(x): + xtype = type(x) + if xtype == self.RTYPE: + if xtype not in cache: + fun = self.compile(f, [x], None) + cache[xtype] = fun + return cache[xtype](x) + return f(x) + super(BaseAdaptedTest,self).unary_test(new_func) + + def binary_test(self, f, rargs = None): + cache = {} + def new_func(x, y): + if type(x) == self.RTYPE or type(y) == self.RTYPE: + types = (type(x), type(y)) + if types not in cache: + fun = self.compile(f, [x, y], None) + cache[types] = fun + return cache[types](x, y) + return f(x,y) + super(BaseAdaptedTest,self).binary_test(new_func, rargs) + +class Test_r_uint(BaseAdaptedTest, BaseTest_r_uint): + RTYPE = ra.r_uint + def test__pow__(self): + py.test.skip("rpython has no ** on ints") + def test__divmod__(self): + # to enable this, you must first replace the call to binary_test(divmod) + # with binary_test(lambda x, y: divmod(x,y)), but even then you run into + # problems + py.test.skip("divmod not fully supported by rtyper") + +#class Test_r_int(BaseAdaptedTest, BaseTest_r_int): +# RTYPE = ra.r_int +# def test__pow__(self): +# py.test.skip("rpython has no ** on ints") +# def test__divmod__(self): +# py.test.skip("divmod not fully supported by rtyper") + From niko at codespeak.net Fri Sep 28 07:16:44 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 28 Sep 2007 07:16:44 +0200 (CEST) Subject: [pypy-svn] r46987 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070928051644.CF8CA80BA@code0.codespeak.net> Author: niko Date: Fri Sep 28 07:16:44 2007 New Revision: 46987 Modified: pypy/dist/pypy/translator/jvm/test/test_backendopt.py Log: adjust backendopt to deal with renamed _compile method (now compile, as it is not really private in any way...) Modified: pypy/dist/pypy/translator/jvm/test/test_backendopt.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_backendopt.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_backendopt.py Fri Sep 28 07:16:44 2007 @@ -5,7 +5,7 @@ class TestOptimizedSwitch(BaseTestOptimizedSwitch): def getcompiled(self, fn, annotation): t = JvmTest() - return t._compile(fn, None, annotation, backendopt=True) + return t.compile(fn, None, annotation, backendopt=True) def test_longlong_switch(self): py.test.skip('fixme!') From arigo at codespeak.net Fri Sep 28 10:36:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 10:36:53 +0200 (CEST) Subject: [pypy-svn] r46988 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070928083653.D7E6C80AE@code0.codespeak.net> Author: arigo Date: Fri Sep 28 10:36:52 2007 New Revision: 46988 Modified: pypy/dist/pypy/module/struct/formatiterator.py pypy/dist/pypy/module/struct/nativefmttable.py pypy/dist/pypy/module/struct/standardfmttable.py pypy/dist/pypy/module/struct/test/test_struct.py Log: Emulate CPython 2.4 accepting broken input for struct.pack(). Modified: pypy/dist/pypy/module/struct/formatiterator.py ============================================================================== --- pypy/dist/pypy/module/struct/formatiterator.py (original) +++ pypy/dist/pypy/module/struct/formatiterator.py Fri Sep 28 10:36:52 2007 @@ -1,9 +1,11 @@ +from pypy.interpreter.error import OperationError from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import ovfcheck from pypy.module.struct.error import StructError from pypy.module.struct.standardfmttable import standard_fmttable +from pypy.module.struct.standardfmttable import PACK_ACCEPTS_BROKEN_INPUT from pypy.module.struct.nativefmttable import native_fmttable from pypy.module.struct.nativefmttable import native_is_bigendian @@ -136,21 +138,63 @@ self.args_index += 1 return w_obj - def accept_int_arg(self): - w_obj = self.accept_obj_arg() - return self.space.int_w(w_obj) - - def accept_uint_arg(self): - w_obj = self.accept_obj_arg() - return self.space.uint_w(w_obj) + if PACK_ACCEPTS_BROKEN_INPUT: + # permissive version - accepts float arguments too - def accept_longlong_arg(self): - w_obj = self.accept_obj_arg() - return self.space.r_longlong_w(w_obj) - - def accept_ulonglong_arg(self): - w_obj = self.accept_obj_arg() - return self.space.r_ulonglong_w(w_obj) + def accept_int_arg(self): + w_obj = self.accept_obj_arg() + try: + return self.space.int_w(w_obj) + except OperationError, e: + return self.space.int_w(self._maybe_float(e, w_obj)) + + def accept_uint_arg(self): + w_obj = self.accept_obj_arg() + try: + return self.space.uint_w(w_obj) + except OperationError, e: + return self.space.uint_w(self._maybe_float(e, w_obj)) + + def accept_longlong_arg(self): + w_obj = self.accept_obj_arg() + try: + return self.space.r_longlong_w(w_obj) + except OperationError, e: + return self.space.r_longlong_w(self._maybe_float(e, w_obj)) + + def accept_ulonglong_arg(self): + w_obj = self.accept_obj_arg() + try: + return self.space.r_ulonglong_w(w_obj) + except OperationError, e: + return self.space.r_ulonglong_w(self._maybe_float(e, w_obj)) + + def _maybe_float(self, e, w_obj): + space = self.space + if not e.match(space, space.w_TypeError): + raise e + if not space.is_true(space.isinstance(w_obj, space.w_float)): + raise e + return space.int(w_obj) # wrapped float -> wrapped int or long + + else: + # strict version + + def accept_int_arg(self): + w_obj = self.accept_obj_arg() + return self.space.int_w(w_obj) + + def accept_uint_arg(self): + w_obj = self.accept_obj_arg() + return self.space.uint_w(w_obj) + + def accept_longlong_arg(self): + w_obj = self.accept_obj_arg() + return self.space.r_longlong_w(w_obj) + + def accept_ulonglong_arg(self): + w_obj = self.accept_obj_arg() + return self.space.r_ulonglong_w(w_obj) def accept_str_arg(self): w_obj = self.accept_obj_arg() Modified: pypy/dist/pypy/module/struct/nativefmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/nativefmttable.py (original) +++ pypy/dist/pypy/module/struct/nativefmttable.py Fri Sep 28 10:36:52 2007 @@ -112,7 +112,8 @@ pack = pack_double unpack = unpack_double else: - pack = std.make_int_packer(size, signed) + cpython_checks_range = fmtchar in 'bBhH' + pack = std.make_int_packer(size, signed, cpython_checks_range) unpack = std.make_int_unpacker(size, signed) native_fmttable[fmtchar] = {'size': size, Modified: pypy/dist/pypy/module/struct/standardfmttable.py ============================================================================== --- pypy/dist/pypy/module/struct/standardfmttable.py (original) +++ pypy/dist/pypy/module/struct/standardfmttable.py Fri Sep 28 10:36:52 2007 @@ -11,6 +11,12 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong +# In the CPython struct module, pack() unconsistently accepts inputs +# that are out-of-range or floats instead of ints. Should we emulate +# this? Let's use a flag for now: + +PACK_ACCEPTS_BROKEN_INPUT = True + # ____________________________________________________________ def pack_pad(fmtiter, count): @@ -59,11 +65,17 @@ native_int_size = struct.calcsize("l") -def make_int_packer(size, signed, _memo={}): +def make_int_packer(size, signed, cpython_checks_range, _memo={}): + if cpython_checks_range: + check_range = True + else: + check_range = not PACK_ACCEPTS_BROKEN_INPUT + key = (size, signed, check_range) try: - return _memo[size, signed] + return _memo[key] except KeyError: pass + if signed: min = -(2 ** (8*size-1)) max = (2 ** (8*size-1)) - 1 @@ -99,8 +111,9 @@ def pack_int(fmtiter): method = getattr(fmtiter, accept_method) value = method() - if value < min or value > max: - raise StructError(errormsg) + if check_range: + if value < min or value > max: + raise StructError(errormsg) if fmtiter.bigendian: for i in unroll_revrange_size: x = (value >> (8*i)) & 0xff @@ -110,7 +123,7 @@ fmtiter.result.append(chr(value & 0xff)) value >>= 8 - _memo[size, signed] = pack_int + _memo[key] = pack_int return pack_int # ____________________________________________________________ @@ -201,8 +214,9 @@ for c, size in [('b', 1), ('h', 2), ('i', 4), ('l', 4), ('q', 8)]: standard_fmttable[c] = {'size': size, - 'pack': make_int_packer(size, True), + 'pack': make_int_packer(size, True, False), 'unpack': make_int_unpacker(size, True)} standard_fmttable[c.upper()] = {'size': size, - 'pack': make_int_packer(size, False), + 'pack': make_int_packer(size, False, + False), 'unpack': make_int_unpacker(size, False)} Modified: pypy/dist/pypy/module/struct/test/test_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_struct.py (original) +++ pypy/dist/pypy/module/struct/test/test_struct.py Fri Sep 28 10:36:52 2007 @@ -310,3 +310,15 @@ raises(someerror, calcsize, "%dcc" % (sys.maxint,)) raises(someerror, calcsize, "c%dc" % (sys.maxint,)) raises(someerror, calcsize, "%dci" % (sys.maxint,)) + + + def test_broken_input(self): + """ + For compatibility: check that we also accept inputs that are + wrongly accepted by CPython 2.4. + """ + pack = self.struct.pack + assert pack("!b", 0xa0) == '\xa0' + assert pack("!B", -1.1) == '\xff' + assert pack("!h", 0xa000) == '\xa0\x00' + assert pack("!H", -2.2) == '\xff\xfe' From arigo at codespeak.net Fri Sep 28 11:09:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 11:09:40 +0200 (CEST) Subject: [pypy-svn] r46989 - in pypy/dist/pypy/rlib: . test Message-ID: <20070928090940.1C8B8813D@code0.codespeak.net> Author: arigo Date: Fri Sep 28 11:09:39 2007 New Revision: 46989 Added: pypy/dist/pypy/rlib/rsha.py - copied, changed from r46987, pypy/dist/pypy/lib/sha.py pypy/dist/pypy/rlib/test/test_rsha.py - copied, changed from r46987, pypy/dist/pypy/lib/app_test/test_sha_extra.py Modified: pypy/dist/pypy/rlib/rmd5.py Log: Ported lib/sha to rlib/rsha while I'm at it. Modified: pypy/dist/pypy/rlib/rmd5.py ============================================================================== --- pypy/dist/pypy/rlib/rmd5.py (original) +++ pypy/dist/pypy/rlib/rmd5.py Fri Sep 28 11:09:39 2007 @@ -67,6 +67,9 @@ ]) def _string2uintlist(s, start=0, count=16): + """Build a list of count r_uint's by unpacking the string + s[start:start+4*count] in little-endian order. + """ result = [] for i in range(count): p = start + i * 4 From arigo at codespeak.net Fri Sep 28 11:10:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 11:10:49 +0200 (CEST) Subject: [pypy-svn] r46990 - pypy/dist/pypy/rlib/test Message-ID: <20070928091049.B8FE9818D@code0.codespeak.net> Author: arigo Date: Fri Sep 28 11:10:49 2007 New Revision: 46990 Modified: pypy/dist/pypy/rlib/test/test_rsha.py Log: A randomized test port from cfbolz in test_rmd5. Modified: pypy/dist/pypy/rlib/test/test_rsha.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsha.py (original) +++ pypy/dist/pypy/rlib/test/test_rsha.py Fri Sep 28 11:10:49 2007 @@ -26,3 +26,14 @@ def disabled_too_slow_test_case_3(self): self.check("a" * 1000000, "34aa973cd4c4daa4f61eeb2bdbad27316534016f") + + def test_random(self): + import random, sha + for i in range(20): + input = ''.join([chr(random.randrange(256)) + for i in range(random.randrange(1000))]) + m1 = rsha.RSHA() + m1.update(input) + m2 = sha.new() + m2.update(input) + assert m2.hexdigest() == m1.hexdigest() From arigo at codespeak.net Fri Sep 28 11:13:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 11:13:26 +0200 (CEST) Subject: [pypy-svn] r46991 - in pypy/dist/pypy/rlib: . test Message-ID: <20070928091326.6E9DC818E@code0.codespeak.net> Author: arigo Date: Fri Sep 28 11:13:26 2007 New Revision: 46991 Modified: pypy/dist/pypy/rlib/rsha.py pypy/dist/pypy/rlib/test/test_rsha.py Log: assert tested or broken Modified: pypy/dist/pypy/rlib/rsha.py ============================================================================== --- pypy/dist/pypy/rlib/rsha.py (original) +++ pypy/dist/pypy/rlib/rsha.py Fri Sep 28 11:13:26 2007 @@ -262,7 +262,7 @@ to efficiently compute the digests of strings that share a common initial substring. """ - clone = RMD5() + clone = RSHA() clone._copyfrom(self) return clone Modified: pypy/dist/pypy/rlib/test/test_rsha.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsha.py (original) +++ pypy/dist/pypy/rlib/test/test_rsha.py Fri Sep 28 11:13:26 2007 @@ -27,6 +27,16 @@ self.check("a" * 1000000, "34aa973cd4c4daa4f61eeb2bdbad27316534016f") + def test_copy(self): + import sha + for repeat in [1, 10, 100]: + d1 = rsha.sha("abc" * repeat) + d2 = d1.copy() + d1.update("def" * repeat) + d2.update("gh" * repeat) + assert d1.digest() == sha.sha("abc"*repeat+"def"*repeat).digest() + assert d2.digest() == sha.sha("abc"*repeat+"gh"*repeat).digest() + def test_random(self): import random, sha for i in range(20): From arigo at codespeak.net Fri Sep 28 11:21:33 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 11:21:33 +0200 (CEST) Subject: [pypy-svn] r46992 - in pypy/dist/pypy/module/sha: . test Message-ID: <20070928092133.7BDA98195@code0.codespeak.net> Author: arigo Date: Fri Sep 28 11:21:33 2007 New Revision: 46992 Added: pypy/dist/pypy/module/sha/ - copied from r46987, pypy/dist/pypy/module/md5/ pypy/dist/pypy/module/sha/interp_sha.py - copied, changed from r46987, pypy/dist/pypy/module/md5/interp_md5.py pypy/dist/pypy/module/sha/test/test_sha.py - copied, changed from r46987, pypy/dist/pypy/module/md5/test/test_md5.py Removed: pypy/dist/pypy/module/sha/interp_md5.py pypy/dist/pypy/module/sha/test/test_md5.py Modified: pypy/dist/pypy/module/sha/__init__.py Log: An interp-level sha module for PyPy, copied from pypy/module/md5. Modified: pypy/dist/pypy/module/sha/__init__.py ============================================================================== --- pypy/dist/pypy/module/md5/__init__.py (original) +++ pypy/dist/pypy/module/sha/__init__.py Fri Sep 28 11:21:33 2007 @@ -1,8 +1,8 @@ """ -Mixed-module definition for the md5 module. -Note that there is also a pure Python implementation in pypy/lib/md5.py; -the present mixed-module version of md5 takes precedence if it is enabled. +Mixed-module definition for the sha module. +Note that there is also a pure Python implementation in pypy/lib/sha.py; +the present mixed-module version of sha takes precedence if it is enabled. """ from pypy.interpreter.mixedmodule import MixedModule @@ -10,19 +10,21 @@ class Module(MixedModule): """\ -This module implements the interface to RSA's MD5 message digest -algorithm (see also Internet RFC 1321). Its use is quite -straightforward: use new() to create an md5 object. You can now feed -this object with arbitrary strings using the update() method, and at any -point you can ask it for the digest (a strong kind of 128-bit checksum, -a.k.a. ``fingerprint'') of the concatenation of the strings fed to it so -far using the digest() method.""" +This module implements the interface to NIST's secure hash algorithm, +known as SHA-1. SHA-1 is an improved version of the original SHA hash +algorithm. It is used in the same way as the md5 module: use new() to +create an sha object, then feed this object with arbitrary strings using +the update() method, and at any point you can ask it for the digest of +the concatenation of the strings fed to it so far. SHA-1 digests are 160 +bits instead of MD5's 128 bits.""" interpleveldefs = { - 'md5': 'interp_md5.W_MD5', - 'new': 'interp_md5.W_MD5', - 'MD5Type': 'interp_md5.W_MD5', - 'digest_size': 'space.wrap(16)', + 'sha': 'interp_sha.W_SHA', + 'new': 'interp_sha.W_SHA', + 'SHAType': 'interp_sha.W_SHA', + 'blocksize': 'space.wrap(1)', + 'digest_size': 'space.wrap(20)', + 'digestsize': 'space.wrap(20)', } appleveldefs = { From arigo at codespeak.net Fri Sep 28 12:14:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 12:14:26 +0200 (CEST) Subject: [pypy-svn] r46999 - in pypy/dist/pypy: config doc/config Message-ID: <20070928101426.B6B258176@code0.codespeak.net> Author: arigo Date: Fri Sep 28 12:14:26 2007 New Revision: 46999 Added: pypy/dist/pypy/doc/config/objspace.usemodules.md5.txt - copied, changed from r46987, pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt pypy/dist/pypy/doc/config/objspace.usemodules.sha.txt - copied, changed from r46987, pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt Modified: pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt Log: The md5 and sha modules seem to work, enable them by default. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Fri Sep 28 12:14:26 2007 @@ -24,7 +24,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", "termios", "zlib", - "struct", + "struct", "md5", "sha", ] )) Modified: pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt (original) +++ pypy/dist/pypy/doc/config/objspace.usemodules.struct.txt Fri Sep 28 12:14:26 2007 @@ -1,4 +1,5 @@ Use the built-in 'struct' module. This module is expected to be working and is included by default. -There is also a pure Python version in pypy/lib, but it is several -orders of magnitude slower. +There is also a pure Python version in pypy/lib which is used +if the built-in is disabled, but it is several orders of magnitude +slower. From arigo at codespeak.net Fri Sep 28 12:15:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 12:15:45 +0200 (CEST) Subject: [pypy-svn] r47000 - in pypy/dist/lib-python: . modified-2.4.1/test Message-ID: <20070928101545.C8989818B@code0.codespeak.net> Author: arigo Date: Fri Sep 28 12:15:45 2007 New Revision: 47000 Modified: pypy/dist/lib-python/conftest.py pypy/dist/lib-python/modified-2.4.1/test/test_struct.py Log: Adaptations for the struct module. Also mention the zlib module. Modified: pypy/dist/lib-python/conftest.py ============================================================================== --- pypy/dist/lib-python/conftest.py (original) +++ pypy/dist/lib-python/conftest.py Fri Sep 28 12:15:45 2007 @@ -679,7 +679,7 @@ RegrTest('test_strptime.py', enabled=False), #rev 10840: 1 of 42 test fails: seems to be some regex problem - RegrTest('test_struct.py', enabled=False, dumbtest=1), + RegrTest('test_struct.py', enabled=True, dumbtest=1, usemodules='struct'), RegrTest('test_structseq.py', enabled=False, dumbtest=1), RegrTest('test_subprocess.py', enabled=False), RegrTest('test_sunaudiodev.py', enabled=False, dumbtest=1), @@ -759,12 +759,8 @@ RegrTest('test_xpickle.py', enabled=False), RegrTest('test_xrange.py', enabled=True, core=True), RegrTest('test_zipfile.py', enabled=False, dumbtest=1), - RegrTest('test_zipimport.py', enabled=True), - # considered non-core because it depends on 'import zlib' - # which we don't have - - RegrTest('test_zlib.py', enabled=False), - #10840: ImportError: zlib + RegrTest('test_zipimport.py', enabled=True, usemodules='zlib'), + RegrTest('test_zlib.py', enabled=True, usemodules='zlib'), ] class RegrDirectory(py.test.collect.Directory): Modified: pypy/dist/lib-python/modified-2.4.1/test/test_struct.py ============================================================================== --- pypy/dist/lib-python/modified-2.4.1/test/test_struct.py (original) +++ pypy/dist/lib-python/modified-2.4.1/test/test_struct.py Fri Sep 28 12:15:45 2007 @@ -30,7 +30,7 @@ def any_err(func, *args): try: func(*args) - except (struct.error, OverflowError, TypeError): + except (struct.error, OverflowError, TypeError, ValueError): pass else: raise TestFailed, "%s%s did not raise error" % ( @@ -53,9 +53,9 @@ simple_err(struct.pack, 'iii', 3) simple_err(struct.pack, 'i', 3, 3, 3) -simple_err(struct.pack, 'i', 'foo') -simple_err(struct.pack, 'P', 'foo') -simple_err(struct.unpack, 'd', 'flap') +any_err(struct.pack, 'i', 'foo') +any_err(struct.pack, 'P', 'foo') +any_err(struct.unpack, 'd', 'flap') s = struct.pack('ii', 1, 2) simple_err(struct.unpack, 'iii', s) simple_err(struct.unpack, 'i', s) @@ -150,8 +150,8 @@ print "Platform has native q/Q?", has_native_qQ and "Yes." or "No." any_err(struct.pack, "Q", -1) # can't pack -1 as unsigned regardless -simple_err(struct.pack, "q", "a") # can't pack string as 'q' regardless -simple_err(struct.pack, "Q", "a") # ditto, but 'Q' +any_err(struct.pack, "q", "a") # can't pack string as 'q' regardless +any_err(struct.pack, "Q", "a") # ditto, but 'Q' def test_native_qQ(): bytes = struct.calcsize('q') From arigo at codespeak.net Fri Sep 28 12:25:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Sep 2007 12:25:47 +0200 (CEST) Subject: [pypy-svn] r47001 - in pypy/dist/pypy/module/struct: . test Message-ID: <20070928102547.8F6DB8176@code0.codespeak.net> Author: arigo Date: Fri Sep 28 12:25:47 2007 New Revision: 47001 Modified: pypy/dist/pypy/module/struct/ieee.py pypy/dist/pypy/module/struct/test/test_ieee.py Log: Test ieee after translation too. Bug fix. Modified: pypy/dist/pypy/module/struct/ieee.py ============================================================================== --- pypy/dist/pypy/module/struct/ieee.py (original) +++ pypy/dist/pypy/module/struct/ieee.py Fri Sep 28 12:25:47 2007 @@ -87,10 +87,12 @@ bias = 1023 exp = 11 prec = 52 + mantissa_scale_factor = 0.5 ** prec # this is constant-folded if it's + # right after the 'if' mantissa = r_longlong(bytes[size-2] & ((1<<(15-exp))-1)) for i in range(size-3, -1, -1): mantissa = mantissa << 8 | bytes[i] - mantissa = 1 + float(mantissa) / (1<> (15 - exp)) & ((1<<(exp - 7)) -1) Modified: pypy/dist/pypy/module/struct/test/test_ieee.py ============================================================================== --- pypy/dist/pypy/module/struct/test/test_ieee.py (original) +++ pypy/dist/pypy/module/struct/test/test_ieee.py Fri Sep 28 12:25:47 2007 @@ -15,16 +15,26 @@ def test_pack(): for number, size, bigendian, expected in testcases: + print 'test_pack:', number, size, bigendian res = [] pack_float(res, number, size, bigendian) - assert res == list(expected) + assert ''.join(res) == expected def test_unpack(): for expected, size, bigendian, input in testcases: + print 'test_unpack:', expected, size, bigendian assert len(input) == size res = unpack_float(input, bigendian) if size == 8: assert res == expected # exact result expected else: assert abs(res - expected) < 1E-6 + + +def test_llinterpreted(): + from pypy.rpython.test.test_llinterp import interpret + def f(): + test_pack() + test_unpack() + interpret(f, []) From pedronis at codespeak.net Sat Sep 29 10:37:54 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 29 Sep 2007 10:37:54 +0200 (CEST) Subject: [pypy-svn] r47009 - in pypy/dist/pypy: bin config doc/config lib/pyontology/test objspace/std objspace/std/test translator/goal Message-ID: <20070929083754.1238E810E@code0.codespeak.net> Author: pedronis Date: Sat Sep 29 10:37:53 2007 New Revision: 47009 Added: pypy/dist/pypy/doc/config/objspace.std.oldstyle.txt - copied unchanged from r46932, pypy/dist/pypy/doc/config/objspace.std.oldstyle.txt Modified: pypy/dist/pypy/bin/py.py pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/lib/pyontology/test/test_sparql.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/test/test_index.py pypy/dist/pypy/translator/goal/app_main.py pypy/dist/pypy/translator/goal/targetpypystandalone.py Log: reverting the old-style change for now (46896, 46933), it broke testing infrastructure and also we probably still want an translation time option to get a pypy-c where old-style are the default. I'll make a branch and re-apply the changes there and try to sort out the testing mess. Modified: pypy/dist/pypy/bin/py.py ============================================================================== --- pypy/dist/pypy/bin/py.py (original) +++ pypy/dist/pypy/bin/py.py Sat Sep 29 10:37:53 2007 @@ -37,9 +37,6 @@ StrOption("runcommand", "program passed in as CMD (terminates option list)", default=None, cmdline="-c"), - BoolOption("oldstyle_classes", - "Use old-style classes by default.", - default=False, cmdline="-k --oldstyle"), ]) pypy_init = gateway.applevel(''' @@ -76,9 +73,11 @@ space = option.make_objspace(config) space._starttime = starttime - space.setitem(space.sys.w_dict, space.wrap('executable'), space.wrap(argv[0])) - if interactiveconfig.oldstyle_classes: - space.setitem(space.builtin.w_dict, space.wrap('__metaclass__'), space.w_classobj) + #assert 'pypy.tool.udir' not in sys.modules, ( + # "running py.py should not import pypy.tool.udir, which is\n" + # "only for testing or translating purposes.") + # ^^^ _socket and other rctypes-based modules need udir + space.setitem(space.sys.w_dict,space.wrap('executable'),space.wrap(argv[0])) # store the command-line arguments into sys.argv go_interactive = interactiveconfig.interactive Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Sat Sep 29 10:37:53 2007 @@ -236,6 +236,10 @@ "special case the 'list[integer]' expressions", default=False), + BoolOption("oldstyle", + "specify whether the default metaclass should be classobj", + default=False, cmdline="--oldstyle"), + BoolOption("logspaceoptypes", "a instrumentation option: before exit, print the types seen by " "certain simpler bytecodes", Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_sparql.py Sat Sep 29 10:37:53 2007 @@ -500,4 +500,4 @@ if check_have_oldstyle(): TestXMLRPC = _TestXMLRPC else: - py.test.skip('Please run PyPy with --oldstyle, needed by xmlrpclib') + py.test.skip('Please build PyPy with --oldstyle, needed by xmlrpclib') Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Sat Sep 29 10:37:53 2007 @@ -48,6 +48,10 @@ PACKAGE_PATH = 'objspace.std' + def setoptions(self, **kwds): + if "oldstyle" in kwds: + self.config.objspace.std.oldstyle = kwds["oldstyle"] + def initialize(self): "NOT_RPYTHON: only for initializing the space." self._typecache = Cache() @@ -249,9 +253,11 @@ """) self.w_dict.__flags__ = old_flags + if self.config.objspace.std.oldstyle: + self.enable_old_style_classes_as_default_metaclass() + # final setup self.setup_builtin_modules() - # Adding transparent proxy call if self.config.objspace.std.withtproxy: w___pypy__ = self.getbuiltinmodule("__pypy__") @@ -262,7 +268,16 @@ self.setattr(w___pypy__, self.wrap('get_tproxy_controller'), self.wrap(app_proxy_controller)) + def enable_old_style_classes_as_default_metaclass(self): + self.setitem(self.builtin.w_dict, self.wrap('__metaclass__'), self.w_classobj) + def enable_new_style_classes_as_default_metaclass(self): + space = self + try: + self.delitem(self.builtin.w_dict, self.wrap('__metaclass__')) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise def setup_old_style_classes(self): """NOT_RPYTHON""" @@ -277,9 +292,6 @@ self.w_classobj = w_classobj self.w_instance = w_instance - def force_old_style_classes(self): - self.setitem(self.builtin.w_dict, self.wrap('__metaclass__'), self.w_classobj) - def create_builtin_module(self, pyname, publicname): """NOT_RPYTHON helper function which returns the wrapped module and its dict. Modified: pypy/dist/pypy/objspace/std/test/test_index.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_index.py (original) +++ pypy/dist/pypy/objspace/std/test/test_index.py Sat Sep 29 10:37:53 2007 @@ -3,9 +3,7 @@ class AppTest_IndexProtocol: def setup_class(self): - self.space = gettestobjspace() - self.space.force_old_style_classes() - + self.space = gettestobjspace(oldstyle=True) w_oldstyle = self.space.appexec([], """(): class oldstyle: def __index__(self): Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Sep 29 10:37:53 2007 @@ -3,16 +3,15 @@ # See test/test_app_main. """ options: - -i inspect interactively after running script - -O dummy optimization flag for compatibility with C Python - -c CMD program passed in as CMD (terminates option list) - -S do not 'import site' on initialization - -u unbuffered binary stdout and stderr - -h, --help show this help message and exit - -m library module to be run as a script (terminates option list) - -k, --oldstyle use old-style classes instead of newstyle classes everywhere - --version print the PyPy version - --info print translation information about this PyPy executable + -i inspect interactively after running script + -O dummy optimization flag for compatibility with C Python + -c CMD program passed in as CMD (terminates option list) + -S do not 'import site' on initialization + -u unbuffered binary stdout and stderr + -h, --help show this help message and exit + -m library module to be run as a script (terminates option list) + --version print the PyPy version + --info print translation information about this PyPy executable """ import sys, os @@ -180,7 +179,6 @@ i = 0 run_module = False run_stdin = False - oldstyle_classes = False while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -218,8 +216,6 @@ return 2 run_module = True break - elif arg in ('-k', '--oldstyle'): - oldstyle_classes = True elif arg == '--': i += 1 break # terminates option list @@ -239,10 +235,6 @@ mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule - if oldstyle_classes: - import __builtin__ - __builtin__.__metaclass__ = __builtin__._classobj - if import_site: try: import site Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Sat Sep 29 10:37:53 2007 @@ -170,6 +170,10 @@ def get_entry_point(self, config): space = make_objspace(config) + if not config.objspace.std.oldstyle: + # disable translation of the whole of classobjinterp.py + StdObjSpace.setup_old_style_classes = lambda self: None + # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') w_dict = space.newdict() From pedronis at codespeak.net Sat Sep 29 10:40:39 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 29 Sep 2007 10:40:39 +0200 (CEST) Subject: [pypy-svn] r47010 - pypy/branch/dynamic-old-style Message-ID: <20070929084039.56452810E@code0.codespeak.net> Author: pedronis Date: Sat Sep 29 10:40:38 2007 New Revision: 47010 Added: pypy/branch/dynamic-old-style/ - copied from r47009, pypy/dist/ Log: making a branch to try to sort out the fall out of the dynamic old-style changes. From pedronis at codespeak.net Sat Sep 29 12:29:15 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 29 Sep 2007 12:29:15 +0200 (CEST) Subject: [pypy-svn] r47011 - pypy/dist/pypy/bin Message-ID: <20070929102915.E2DF2810D@code0.codespeak.net> Author: pedronis Date: Sat Sep 29 12:29:14 2007 New Revision: 47011 Modified: pypy/dist/pypy/bin/py.py Log: redo xoraxax cleanup Modified: pypy/dist/pypy/bin/py.py ============================================================================== --- pypy/dist/pypy/bin/py.py (original) +++ pypy/dist/pypy/bin/py.py Sat Sep 29 12:29:14 2007 @@ -73,11 +73,8 @@ space = option.make_objspace(config) space._starttime = starttime - #assert 'pypy.tool.udir' not in sys.modules, ( - # "running py.py should not import pypy.tool.udir, which is\n" - # "only for testing or translating purposes.") - # ^^^ _socket and other rctypes-based modules need udir - space.setitem(space.sys.w_dict,space.wrap('executable'),space.wrap(argv[0])) + space.setitem(space.sys.w_dict, space.wrap('executable'), + space.wrap(argv[0])) # store the command-line arguments into sys.argv go_interactive = interactiveconfig.interactive From cfbolz at codespeak.net Sat Sep 29 12:49:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 12:49:30 +0200 (CEST) Subject: [pypy-svn] r47013 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929104930.1665B810E@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 12:49:29 2007 New Revision: 47013 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: resolve XXX in sprint announcement Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 29 12:49:29 2007 @@ -1,7 +1,10 @@ PyPy G?teborg Cleanup Sprint November 19 - November 25 2007 =========================================================== -XXX +The first post-EU-project PyPy sprint will be in Gothenburg, Sweden. It will +focus on cleaning up the PyPy codebase and making it ready for the next round +of improvements. It is a "public" sprint but it will probably be more suitable +for people already somewhat acquainted with PyPy. Topics and goals ---------------- From pedronis at codespeak.net Sat Sep 29 12:49:43 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 29 Sep 2007 12:49:43 +0200 (CEST) Subject: [pypy-svn] r47014 - in pypy/branch/dynamic-old-style/pypy: config lib/pyontology/test translator/goal Message-ID: <20070929104943.0C4C5810D@code0.codespeak.net> Author: pedronis Date: Sat Sep 29 12:49:43 2007 New Revision: 47014 Modified: pypy/branch/dynamic-old-style/pypy/config/pypyoption.py pypy/branch/dynamic-old-style/pypy/lib/pyontology/test/test_sparql.py pypy/branch/dynamic-old-style/pypy/translator/goal/app_main.py pypy/branch/dynamic-old-style/pypy/translator/goal/targetpypystandalone.py Log: pare down 46896 dynamic old-style changes to be much more backward compatible for the sake of lib-python/conftest.py and the nightly pypy-c testing Modified: pypy/branch/dynamic-old-style/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/dynamic-old-style/pypy/config/pypyoption.py (original) +++ pypy/branch/dynamic-old-style/pypy/config/pypyoption.py Sat Sep 29 12:49:43 2007 @@ -238,7 +238,7 @@ BoolOption("oldstyle", "specify whether the default metaclass should be classobj", - default=False, cmdline="--oldstyle"), + default=False, cmdline="-k --oldstyle"), BoolOption("logspaceoptypes", "a instrumentation option: before exit, print the types seen by " Modified: pypy/branch/dynamic-old-style/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/branch/dynamic-old-style/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/branch/dynamic-old-style/pypy/lib/pyontology/test/test_sparql.py Sat Sep 29 12:49:43 2007 @@ -500,4 +500,4 @@ if check_have_oldstyle(): TestXMLRPC = _TestXMLRPC else: - py.test.skip('Please build PyPy with --oldstyle, needed by xmlrpclib') + py.test.skip('Please run PyPy with --oldstyle, needed by xmlrpclib') Modified: pypy/branch/dynamic-old-style/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/dynamic-old-style/pypy/translator/goal/app_main.py (original) +++ pypy/branch/dynamic-old-style/pypy/translator/goal/app_main.py Sat Sep 29 12:49:43 2007 @@ -3,15 +3,17 @@ # See test/test_app_main. """ options: - -i inspect interactively after running script - -O dummy optimization flag for compatibility with C Python - -c CMD program passed in as CMD (terminates option list) - -S do not 'import site' on initialization - -u unbuffered binary stdout and stderr - -h, --help show this help message and exit - -m library module to be run as a script (terminates option list) - --version print the PyPy version - --info print translation information about this PyPy executable + -i inspect interactively after running script + -O dummy optimization flag for compatibility with C Python + -c CMD program passed in as CMD (terminates option list) + -S do not 'import site' on initialization + -u unbuffered binary stdout and stderr + -h, --help show this help message and exit + -m library module to be run as a script (terminates option list) + -k, --oldstyle use old-style classes instead of newstyle classes + everywhere %(oldstyle)s + --version print the PyPy version + --info print translation information about this PyPy executable """ import sys, os @@ -121,7 +123,10 @@ def print_help(): print 'usage: %s [options]' % (sys.executable,) - print __doc__ + details = {'oldstyle': ''} + if sys.pypy_translation_info['objspace.std.oldstyle']: + details['oldstyle'] = '[default]' + print __doc__ % details def print_error(msg): print >> sys.stderr, msg @@ -179,6 +184,7 @@ i = 0 run_module = False run_stdin = False + oldstyle_classes = False while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -216,6 +222,8 @@ return 2 run_module = True break + elif arg in ('-k', '--oldstyle'): + oldstyle_classes = True elif arg == '--': i += 1 break # terminates option list @@ -235,6 +243,10 @@ mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule + if oldstyle_classes: + import __builtin__ + __builtin__.__metaclass__ = __builtin__._classobj + if import_site: try: import site Modified: pypy/branch/dynamic-old-style/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/dynamic-old-style/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/dynamic-old-style/pypy/translator/goal/targetpypystandalone.py Sat Sep 29 12:49:43 2007 @@ -170,10 +170,6 @@ def get_entry_point(self, config): space = make_objspace(config) - if not config.objspace.std.oldstyle: - # disable translation of the whole of classobjinterp.py - StdObjSpace.setup_old_style_classes = lambda self: None - # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') w_dict = space.newdict() From cfbolz at codespeak.net Sat Sep 29 12:53:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 12:53:11 +0200 (CEST) Subject: [pypy-svn] r47015 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929105311.B81F9810E@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 12:53:11 2007 New Revision: 47015 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: some more changes Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 29 12:53:11 2007 @@ -20,7 +20,6 @@ - finish rctypes removal - - implement correct implicit GIL release - move away from implicit keepalives - raw_malloc should be split, rffi one should not use obmalloc (it's @@ -55,7 +54,7 @@ - there's a shadow tracking bug it seems - - fix taint space + - fix the taint space - review the things implemented at applevel whether they are performance-critical @@ -69,7 +68,7 @@ Location -------- - +XXX Registration ------------ From pedronis at codespeak.net Sat Sep 29 12:57:12 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 29 Sep 2007 12:57:12 +0200 (CEST) Subject: [pypy-svn] r47016 - in pypy/dist/pypy: config lib/pyontology/test translator/goal Message-ID: <20070929105712.3E2C5810E@code0.codespeak.net> Author: pedronis Date: Sat Sep 29 12:57:11 2007 New Revision: 47016 Modified: pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/lib/pyontology/test/test_sparql.py pypy/dist/pypy/translator/goal/app_main.py pypy/dist/pypy/translator/goal/targetpypystandalone.py Log: merging dynamic-old-style branch where xoraxax 46896 was pared down to be more backward compatible for the sake of nigthly pypy-c testing and lib-python/conftest.py Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Sat Sep 29 12:57:11 2007 @@ -238,7 +238,7 @@ BoolOption("oldstyle", "specify whether the default metaclass should be classobj", - default=False, cmdline="--oldstyle"), + default=False, cmdline="-k --oldstyle"), BoolOption("logspaceoptypes", "a instrumentation option: before exit, print the types seen by " Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_sparql.py Sat Sep 29 12:57:11 2007 @@ -500,4 +500,4 @@ if check_have_oldstyle(): TestXMLRPC = _TestXMLRPC else: - py.test.skip('Please build PyPy with --oldstyle, needed by xmlrpclib') + py.test.skip('Please run PyPy with --oldstyle, needed by xmlrpclib') Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Sep 29 12:57:11 2007 @@ -3,15 +3,17 @@ # See test/test_app_main. """ options: - -i inspect interactively after running script - -O dummy optimization flag for compatibility with C Python - -c CMD program passed in as CMD (terminates option list) - -S do not 'import site' on initialization - -u unbuffered binary stdout and stderr - -h, --help show this help message and exit - -m library module to be run as a script (terminates option list) - --version print the PyPy version - --info print translation information about this PyPy executable + -i inspect interactively after running script + -O dummy optimization flag for compatibility with C Python + -c CMD program passed in as CMD (terminates option list) + -S do not 'import site' on initialization + -u unbuffered binary stdout and stderr + -h, --help show this help message and exit + -m library module to be run as a script (terminates option list) + -k, --oldstyle use old-style classes instead of newstyle classes + everywhere %(oldstyle)s + --version print the PyPy version + --info print translation information about this PyPy executable """ import sys, os @@ -121,7 +123,10 @@ def print_help(): print 'usage: %s [options]' % (sys.executable,) - print __doc__ + details = {'oldstyle': ''} + if sys.pypy_translation_info['objspace.std.oldstyle']: + details['oldstyle'] = '[default]' + print __doc__ % details def print_error(msg): print >> sys.stderr, msg @@ -179,6 +184,7 @@ i = 0 run_module = False run_stdin = False + oldstyle_classes = False while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -216,6 +222,8 @@ return 2 run_module = True break + elif arg in ('-k', '--oldstyle'): + oldstyle_classes = True elif arg == '--': i += 1 break # terminates option list @@ -235,6 +243,10 @@ mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule + if oldstyle_classes: + import __builtin__ + __builtin__.__metaclass__ = __builtin__._classobj + if import_site: try: import site Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Sat Sep 29 12:57:11 2007 @@ -170,10 +170,6 @@ def get_entry_point(self, config): space = make_objspace(config) - if not config.objspace.std.oldstyle: - # disable translation of the whole of classobjinterp.py - StdObjSpace.setup_old_style_classes = lambda self: None - # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') w_dict = space.newdict() From cfbolz at codespeak.net Sat Sep 29 13:13:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 13:13:05 +0200 (CEST) Subject: [pypy-svn] r47017 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929111305.4CAFB8115@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 13:13:03 2007 New Revision: 47017 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: it's not the first post-funding sprint (thanks anto) Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 29 13:13:03 2007 @@ -1,7 +1,7 @@ PyPy G?teborg Cleanup Sprint November 19 - November 25 2007 =========================================================== -The first post-EU-project PyPy sprint will be in Gothenburg, Sweden. It will +The next post-EU-project PyPy sprint will be in Gothenburg, Sweden. It will focus on cleaning up the PyPy codebase and making it ready for the next round of improvements. It is a "public" sprint but it will probably be more suitable for people already somewhat acquainted with PyPy. From cfbolz at codespeak.net Sat Sep 29 15:13:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 15:13:53 +0200 (CEST) Subject: [pypy-svn] r47018 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929131353.F35AD8106@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 15:13:52 2007 New Revision: 47018 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: add something about the location Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 29 15:13:52 2007 @@ -68,7 +68,8 @@ Location -------- -XXX +The sprint will take place in the apartment of Laura Creighton and Jacob Hall?n +in Gothenburg. Registration ------------ From cfbolz at codespeak.net Sat Sep 29 15:24:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 15:24:59 +0200 (CEST) Subject: [pypy-svn] r47019 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929132459.19D368104@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 15:24:56 2007 New Revision: 47019 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: a more elaborate location description Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 29 15:24:56 2007 @@ -68,8 +68,21 @@ Location -------- -The sprint will take place in the apartment of Laura Creighton and Jacob Hall?n -in Gothenburg. +The sprint will be held in the apartment of Laura Creighton and Jacob Hall?n +which is at G?tabergsgatan 22 in Gothenburg, Sweden. Here is a map_. This is +in central Gothenburg. It is between the tram_ stops of Vasaplatsen and Valand, +where many lines call. + +.. _tram: http://www.vasttrafik.se + +.. _map: http://gulasidorna.eniro.se/query?stq=0&streetname=G%F6tabergsgatan+22%2C+G%F6teborg&what=map&asearch=1 + +Probably cheapest and not too far away is to book accomodation at `SGS +Veckobostader`_. There are also hotels to suit every price range and desire for +luxury. Just ask in the mailing list. + +.. _`SGS Veckobostader`: http://www.sgsveckobostader.com + Registration ------------ From cfbolz at codespeak.net Sat Sep 29 15:36:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 15:36:35 +0200 (CEST) Subject: [pypy-svn] r47020 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929133635.A56358106@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 15:36:35 2007 New Revision: 47020 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: kill sentence about Bern (!) which is redundant anyway, since we are talking about hotels above. Thanks Samuele. Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 29 15:36:35 2007 @@ -91,9 +91,7 @@ and drop a note about your interests and post any questions. More organisational information will be send to that list. We'll keep a list of `people`_ which we'll update (which you can do so yourself if you -have codespeak commit rights). If you have no clue where to stay in -Bern ask around on the list, maybe we can help you find something or -find some other solution. +have codespeak commit rights). .. _`pypy-sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint .. _`people`: http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2007/people.html From lac at codespeak.net Sat Sep 29 17:58:02 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sat, 29 Sep 2007 17:58:02 +0200 (CEST) Subject: [pypy-svn] r47021 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929155802.4A3D68106@code0.codespeak.net> Author: lac Date: Sat Sep 29 17:58:01 2007 New Revision: 47021 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: Those of us who live here won't be arriving Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sat Sep 29 17:58:01 2007 @@ -9,6 +9,11 @@ Name Arrive/Depart Accomodation ==================== ============== ===================== Carl Friedrich Bolz ? ? +Samuele Pedroni lives in G?teborg +Anders Chrigstrom lives in G?teborg +Jacob Hallen lives in G?teborg +Laura Creighton lives in G?teborg +Christian Tismer Laura and Jacob's House ==================== ============== ===================== @@ -20,7 +25,6 @@ Armin Rigo ? ? Michael Hudson ? ? Antonio Cuni ? ? -Samuele Pedroni ? ? Anders Chrigstroem ? ? Maciej Fijalkowski ? ? Andrew Thompson ? ? From cfbolz at codespeak.net Sat Sep 29 18:41:31 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 18:41:31 +0200 (CEST) Subject: [pypy-svn] r47022 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929164131.DC04C810D@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 18:41:30 2007 New Revision: 47022 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: add the date Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sat Sep 29 18:41:30 2007 @@ -1,5 +1,5 @@ -People coming to the Gothenburg sprint November 2007 -==================================================== +People coming to the Gothenburg sprint Nov 19-25 2007 +====================================================== People who have a ``?`` in their arrive/depart or accomodation column are known to be coming but there are no details From cfbolz at codespeak.net Sat Sep 29 18:46:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 18:46:28 +0200 (CEST) Subject: [pypy-svn] r47023 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929164628.6FE03810A@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 18:46:27 2007 New Revision: 47023 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: use iso encoding instead of utf-8. I can't get rest to do the right thing with utf-8, right now. Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sat Sep 29 18:46:27 2007 @@ -1,5 +1,5 @@ -PyPy G??teborg Cleanup Sprint November 19 - November 25 2007 -=========================================================== +PyPy G?teborg Cleanup Sprint November 19 - November 25 2007 +=============================================================== The next post-EU-project PyPy sprint will be in Gothenburg, Sweden. It will focus on cleaning up the PyPy codebase and making it ready for the next round @@ -68,8 +68,8 @@ Location -------- -The sprint will be held in the apartment of Laura Creighton and Jacob Hall??n -which is at G??tabergsgatan 22 in Gothenburg, Sweden. Here is a map_. This is +The sprint will be held in the apartment of Laura Creighton and Jacob Hall?n +which is at G?tabergsgatan 22 in Gothenburg, Sweden. Here is a map_. This is in central Gothenburg. It is between the tram_ stops of Vasaplatsen and Valand, where many lines call. From cfbolz at codespeak.net Sat Sep 29 19:11:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 19:11:59 +0200 (CEST) Subject: [pypy-svn] r47024 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070929171159.2D6EC8106@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 19:11:57 2007 New Revision: 47024 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: fix table Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sat Sep 29 19:11:57 2007 @@ -9,11 +9,11 @@ Name Arrive/Depart Accomodation ==================== ============== ===================== Carl Friedrich Bolz ? ? -Samuele Pedroni lives in G?teborg -Anders Chrigstrom lives in G?teborg -Jacob Hallen lives in G?teborg -Laura Creighton lives in G?teborg -Christian Tismer Laura and Jacob's House +Samuele Pedroni lives there +Anders Chrigstrom lives there +Jacob Hallen lives there +Laura Creighton lives there +Christian Tismer Laura and Jacob's ==================== ============== ===================== From cfbolz at codespeak.net Sat Sep 29 19:13:02 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 29 Sep 2007 19:13:02 +0200 (CEST) Subject: [pypy-svn] r47025 - pypy/dist/pypy/doc Message-ID: <20070929171302.4842B8111@code0.codespeak.net> Author: cfbolz Date: Sat Sep 29 19:13:01 2007 New Revision: 47025 Modified: pypy/dist/pypy/doc/news.txt Log: sprint news item Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Sat Sep 29 19:13:01 2007 @@ -15,6 +15,24 @@ .. _eventhistory: eventhistory.html + +PyPy Sprint in Gothenburg: 19nd-25th November 2007 +================================================================== + + +The next post-EU-project PyPy sprint will be in Gothenburg, Sweden. It will +focus on cleaning up the PyPy codebase and making it ready for the next round +of improvements. It is a "public" sprint but it will probably be more suitable +for people already somewhat acquainted with PyPy. For more information see the +`Gothenburg sprint announcement`_ or a list of the `people that are known to +come to Gothenburg`_. + +.. _`Gothenburg sprint announcement`: http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2007/announce.html +.. _`people that are known to come to Gothenburg`: http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2007/people.html + + + + PyPy/Squeak Sprint in Bern: 22nd-26th October 2007 ================================================================== From arigo at codespeak.net Sat Sep 29 19:31:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 Sep 2007 19:31:18 +0200 (CEST) Subject: [pypy-svn] r47026 - in pypy/dist/pypy: rpython rpython/lltypesystem/test rpython/memory/gctransform translator/c translator/c/src Message-ID: <20070929173118.3983E8106@code0.codespeak.net> Author: arigo Date: Sat Sep 29 19:31:17 2007 New Revision: 47026 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rcpyclass.py pypy/dist/pypy/rpython/memory/gctransform/refcounting.py pypy/dist/pypy/rpython/rcpy.py pypy/dist/pypy/translator/c/funcgen.py pypy/dist/pypy/translator/c/src/mem.h Log: Support Python subclasses of types created by the extension compiler. Don't think I'm actively developping the extension compiler - I was fixing the failures in test_rcpyclass and I was forced to do the right thing :-) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rcpyclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rcpyclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rcpyclass.py Sat Sep 29 19:31:17 2007 @@ -14,7 +14,7 @@ return self.x * 2 -mytest = CPyTypeInterface('mytest', {}) +mytest = CPyTypeInterface('mytest', {}, subclassable=True) def test_cpy_export(): def f(): @@ -22,7 +22,8 @@ return cpy_export(mytest, w) fn = compile(f, []) - res = fn(expected_extra_mallocs=1) + res = fn() # the W_MyTest is allocated with the CPython logic, + # so it doesn't count in expected_extra_mallocs assert type(res).__name__ == 'mytest' @@ -76,17 +77,17 @@ return cpy_export(mytest, w), total fn = compile(f, [object]) - obj, total = fn(None, expected_extra_mallocs=2) # 1 W_MyTest (with 1 tuple) + obj, total = fn(None, expected_extra_mallocs=1) # 1 tuple in w.stuff assert total == 1 - obj, total = fn(obj, expected_extra_mallocs=4) # 2 W_MyTests alive - assert total == 3 - obj, total = fn(obj, expected_extra_mallocs=4) # 2 W_MyTests alive + obj, total = fn(obj, expected_extra_mallocs=2) # 2 tuples: old obj.stuff + assert total == 3 # + new obj.stuff + obj, total = fn(obj, expected_extra_mallocs=2) assert total == 6 - obj, total = fn(obj, expected_extra_mallocs=4) # etc + obj, total = fn(obj, expected_extra_mallocs=2) # idem assert total == 10 - obj, total = fn(obj, expected_extra_mallocs=4) + obj, total = fn(obj, expected_extra_mallocs=2) assert total == 15 - obj, total = fn(obj, expected_extra_mallocs=4) + obj, total = fn(obj, expected_extra_mallocs=2) assert total == 21 @@ -100,32 +101,32 @@ return cpy_export(mytest, w), w.x fn = compile(f, [object]) - obj, x = fn(None, expected_extra_mallocs=1) # 1 W_MyTest + obj, x = fn(None) assert x == 22 obj2 = type(obj)() del obj - obj, x = fn(obj2, expected_extra_mallocs=1) # 1 W_MyTest (obj2) + obj, x = fn(obj2) assert obj is obj2 assert x == 601 # 600 is the class default of W_MyTest.x def test_subclass_from_cpython(): - import py; py.test.skip("not implemented (see comments in rcpy.py)") - def f(input): current = total = 10 if input: w = cpy_import(W_MyTest, input) - current, total = w.stuff + current = w.current # or 0 if left uninitialized, as by U() + total = w.total # or 0 if left uninitialized, as by U() w = W_MyTest(21) current += 1 total += current - w.stuff = current, total + w.current = current + w.total = total return cpy_export(mytest, w), total fn = compile(f, [object]) - obj, total = fn(None, expected_extra_mallocs=2) # 1 W_MyTest (with 1 tuple) + obj, total = fn(None) assert total == 21 T = type(obj) class U(T): @@ -136,11 +137,11 @@ del obj objlist = [U() for i in range(100)] - obj, total = fn(obj2, expected_extra_mallocs=204) # 102 W_MyTests alive + obj, total = fn(obj2) assert total == 1 del objlist - obj, total = fn(obj, expected_extra_mallocs=6) # 3 W_MyTests alive + obj, total = fn(obj) assert total == 3 @@ -151,7 +152,7 @@ return cpy_export(mytest2, w) fn = compile(f, []) - obj = fn(expected_extra_mallocs=1) + obj = fn() assert obj.hi == 123 assert type(obj).hi == 123 @@ -164,7 +165,7 @@ return cpy_export(mytest2, w) fn = compile(f, []) - obj = fn(expected_extra_mallocs=1) + obj = fn() assert obj.hi == 123 assert type(obj).hi == 123 assert obj.there == "foo" Modified: pypy/dist/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/refcounting.py Sat Sep 29 19:31:17 2007 @@ -69,6 +69,8 @@ gcheader.refcount = refcount def ll_no_pointer_dealloc(adr): llop.gc_free(lltype.Void, adr) + def ll_no_pointer_cpydealloc(adr): + llop.cpy_free(lltype.Void, adr) mh = mallocHelpers() mh.allocate = lladdress.raw_malloc @@ -102,6 +104,8 @@ ll_decref_simple, [llmemory.Address], lltype.Void) self.no_pointer_dealloc_ptr = self.inittime_helper( ll_no_pointer_dealloc, [llmemory.Address], lltype.Void) + self.no_pointer_cpydealloc_ptr = self.inittime_helper( + ll_no_pointer_cpydealloc, [llmemory.Address], lltype.Void) self.malloc_fixedsize_ptr = self.inittime_helper( ll_malloc_fixedsize, [lltype.Signed], llmemory.Address) self.malloc_varsize_no_length_ptr = self.inittime_helper( @@ -195,7 +199,10 @@ if destrptr is None and not find_gc_ptrs_in_type(TYPE): #print repr(TYPE)[:80], 'is dealloc easy' - p = self.no_pointer_dealloc_ptr.value + if TYPE._gckind == 'cpy': + p = self.no_pointer_cpydealloc_ptr.value + else: + p = self.no_pointer_dealloc_ptr.value self.static_deallocator_funcptrs[TYPE] = p return p @@ -215,19 +222,19 @@ gcheader.refcount = refcount if refcount == 0: %s - llop.gc_free(lltype.Void, addr) + llop.%s_free(lltype.Void, addr) except: pass llop.gc_restore_exception(lltype.Void, exc_instance) pop_alive(exc_instance) # XXX layering of exceptiontransform versus gcpolicy -""" % (body, ) +""" % (body, TYPE._gckind) else: call_del = None body = '\n'.join(_static_deallocator_body_for_type('v', TYPE)) src = ('def ll_deallocator(addr):\n v = cast_adr_to_ptr(addr, PTR_TYPE)\n' + - body + '\n llop.gc_free(lltype.Void, addr)\n') + body + '\n llop.%s_free(lltype.Void, addr)\n' % (TYPE._gckind,)) d = {'pop_alive': LLTransformerOp(self.pop_alive), 'llop': llop, 'lltype': lltype, Modified: pypy/dist/pypy/rpython/rcpy.py ============================================================================== --- pypy/dist/pypy/rpython/rcpy.py (original) +++ pypy/dist/pypy/rpython/rcpy.py Sat Sep 29 19:31:17 2007 @@ -264,7 +264,7 @@ p[len(name)] = '\x00' pytypeobj.c_tp_name = lltype.direct_arrayitems(p) pytypeobj.c_tp_basicsize = llmemory.sizeof(r_inst.lowleveltype.TO) - if cpytype.subclassable and False: # XXX deallocation of subclass object segfaults! + if cpytype.subclassable: pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES | Py_TPFLAGS_BASETYPE)''') else: @@ -302,13 +302,6 @@ cache[r_inst.classdef] = result return result -# To make this a Py_TPFLAGS_BASETYPE, we need to have a tp_new that does -# something different for subclasses: it needs to allocate a bit more -# for CPython's GC (see PyObject_GC_Malloc); it needs to Py_INCREF the -# type if it's a heap type; and it needs to PyObject_GC_Track() the object. -# Also, tp_dealloc needs to untrack the object. - - # ____________________________________________________________ # Emulation support, to have user-defined classes and instances # work nicely on top of CPython running the CPyObjSpace Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Sat Sep 29 19:31:17 2007 @@ -538,7 +538,7 @@ return "OP_CPY_MALLOC(%s, %s, %s);" % (cpytype, eresult, erestype) def OP_CPY_FREE(self, op): - return "OP_CPY_FREE(%s)" % (self.expr(op.args[1]),) + return "OP_CPY_FREE(%s);" % (self.expr(op.args[0]),) def OP_DIRECT_FIELDPTR(self, op): return self.OP_GETFIELD(op, ampersand='&') Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Sat Sep 29 19:31:17 2007 @@ -115,10 +115,11 @@ #define OP_CPY_MALLOC(cpytype, r, restype) { \ /* XXX add tp_itemsize later */ \ - r = ((PyTypeObject *)cpytype)->tp_alloc((PyTypeObject *)cpytype, 0); \ + r = (restype)((PyTypeObject *)cpytype)->tp_alloc( \ + (PyTypeObject *)cpytype, 0); \ if (!r) RPyConvertExceptionFromCPython(); \ } -#define OP_CPY_FREE(x) XXX "this shouldn't be used any more" +#define OP_CPY_FREE(x) ((PyObject *)x)->ob_type->tp_free((PyObject *)x) /************************************************************/ /* weakref support */ From arigo at codespeak.net Sun Sep 30 11:11:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Sep 2007 11:11:37 +0200 (CEST) Subject: [pypy-svn] r47030 - pypy/extradoc/sprintinfo/bern2007 Message-ID: <20070930091137.6404E80EB@code0.codespeak.net> Author: arigo Date: Sun Sep 30 11:11:35 2007 New Revision: 47030 Modified: pypy/extradoc/sprintinfo/bern2007/people.txt Log: Add myself. No dates yet Modified: pypy/extradoc/sprintinfo/bern2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/bern2007/people.txt (original) +++ pypy/extradoc/sprintinfo/bern2007/people.txt Sun Sep 30 11:11:35 2007 @@ -11,6 +11,7 @@ Carl Friedrich Bolz ? ? Adrian Kuhn - private Niko Matsakis 22.10 / 26.10 private (Zurich) +Armin Rigo ? ? ==================== ============== ===================== @@ -19,7 +20,6 @@ ==================== ============== ===================== Name Arrive/Depart Accomodation ==================== ============== ===================== -Armin Rigo ? ? Michael Hudson ? ? Antonio Cuni ? ? Samuele Pedroni ? ? From fijal at codespeak.net Sun Sep 30 11:37:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 11:37:20 +0200 (CEST) Subject: [pypy-svn] r47031 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070930093720.1AF7A8103@code0.codespeak.net> Author: fijal Date: Sun Sep 30 11:37:19 2007 New Revision: 47031 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: Added myself, no dates yet. Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sun Sep 30 11:37:19 2007 @@ -14,6 +14,7 @@ Jacob Hallen lives there Laura Creighton lives there Christian Tismer Laura and Jacob's +Maciej Fijalkowski ? ? ==================== ============== ===================== @@ -26,7 +27,6 @@ Michael Hudson ? ? Antonio Cuni ? ? Anders Chrigstroem ? ? -Maciej Fijalkowski ? ? Andrew Thompson ? ? Stephan Diehl ? ? Eric van Riet paap ? ? From cfbolz at codespeak.net Sun Sep 30 13:45:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 30 Sep 2007 13:45:46 +0200 (CEST) Subject: [pypy-svn] r47033 - pypy/dist/pypy/doc Message-ID: <20070930114546.8E7F580EA@code0.codespeak.net> Author: cfbolz Date: Sun Sep 30 13:45:45 2007 New Revision: 47033 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: list of rctypes-modules that need to be rewritten. Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Sun Sep 30 13:45:45 2007 @@ -32,4 +32,15 @@ - review the things implemented at applevel whether they are performance- critical - - review threading (especially GIL). It's segfaulting quite often now. + - rewrite the following rctypes modules using rffi: + - bz2 + - crypt + - _ssl + - errno + - fcntl + - select + - mmap + - rctime + - readlione + - select + - signal From cfbolz at codespeak.net Sun Sep 30 13:48:26 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 30 Sep 2007 13:48:26 +0200 (CEST) Subject: [pypy-svn] r47034 - pypy/dist/pypy/doc Message-ID: <20070930114826.2E95080EA@code0.codespeak.net> Author: cfbolz Date: Sun Sep 30 13:48:25 2007 New Revision: 47034 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: double entry and typo. thanks samuele. Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Sun Sep 30 13:48:25 2007 @@ -38,9 +38,8 @@ - _ssl - errno - fcntl - - select - mmap - rctime - - readlione + - readline - select - signal From cfbolz at codespeak.net Sun Sep 30 13:53:37 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 30 Sep 2007 13:53:37 +0200 (CEST) Subject: [pypy-svn] r47035 - pypy/dist/pypy/doc Message-ID: <20070930115337.2626D80EA@code0.codespeak.net> Author: cfbolz Date: Sun Sep 30 13:53:36 2007 New Revision: 47035 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: add something about the extcompiler Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Sun Sep 30 13:53:36 2007 @@ -43,3 +43,5 @@ - readline - select - signal + + - rewrite or kill the extcompiler From pedronis at codespeak.net Sun Sep 30 13:55:45 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 30 Sep 2007 13:55:45 +0200 (CEST) Subject: [pypy-svn] r47036 - pypy/branch/dynamic-old-style Message-ID: <20070930115545.9015780EA@code0.codespeak.net> Author: pedronis Date: Sun Sep 30 13:55:45 2007 New Revision: 47036 Removed: pypy/branch/dynamic-old-style/ Log: kill merged branch From fijal at codespeak.net Sun Sep 30 13:55:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 13:55:52 +0200 (CEST) Subject: [pypy-svn] r47037 - pypy/dist/pypy/rlib Message-ID: <20070930115552.B36C680FC@code0.codespeak.net> Author: fijal Date: Sun Sep 30 13:55:52 2007 New Revision: 47037 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py Log: This hack seems to be no longer needed Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Sun Sep 30 13:55:52 2007 @@ -297,17 +297,6 @@ sockaddr_ptr.TO.become(cConfig.sockaddr) addrinfo_ptr.TO.become(cConfig.addrinfo) -# HACK HACK HACK -if _MS_WINDOWS: - pass #XXX - #from ctypes import Structure - #for struct in cConfig.__dict__.values(): - # if isinstance(struct, type) and issubclass(struct, Structure): - # if struct.__name__ == 'in6_addr': - # struct.__name__ = '_in6_addr' - # else: - # struct._external_ = True # hack to avoid redeclaration of the struct in C - # fill in missing constants with reasonable defaults cConfig.NI_MAXHOST = cConfig.NI_MAXHOST or 1025 cConfig.NI_MAXSERV = cConfig.NI_MAXSERV or 32 From fijal at codespeak.net Sun Sep 30 13:57:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 13:57:16 +0200 (CEST) Subject: [pypy-svn] r47038 - pypy/dist/pypy/rlib Message-ID: <20070930115716.8EB2880F5@code0.codespeak.net> Author: fijal Date: Sun Sep 30 13:57:16 2007 New Revision: 47038 Modified: pypy/dist/pypy/rlib/getaddrinfo.py pypy/dist/pypy/rlib/getnameinfo.py Log: fix docstrings Modified: pypy/dist/pypy/rlib/getaddrinfo.py ============================================================================== --- pypy/dist/pypy/rlib/getaddrinfo.py (original) +++ pypy/dist/pypy/rlib/getaddrinfo.py Sun Sep 30 13:57:16 2007 @@ -1,5 +1,5 @@ """ -An RPython implementation of getaddrinfo() based on ctypes. +An RPython implementation of getaddrinfo(). This is a rewrite of the CPython source: Modules/getaddrinfo.c """ Modified: pypy/dist/pypy/rlib/getnameinfo.py ============================================================================== --- pypy/dist/pypy/rlib/getnameinfo.py (original) +++ pypy/dist/pypy/rlib/getnameinfo.py Sun Sep 30 13:57:16 2007 @@ -1,5 +1,5 @@ """ -An RPython implementation of getnameinfo() based on ctypes. +An RPython implementation of getnameinfo(). This is a rewrite of the CPython source: Modules/getaddrinfo.c """ from pypy.rlib import _rsocket_rffi as _c From fijal at codespeak.net Sun Sep 30 14:03:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 14:03:32 +0200 (CEST) Subject: [pypy-svn] r47039 - pypy/dist/pypy/doc Message-ID: <20070930120332.E38F480CC@code0.codespeak.net> Author: fijal Date: Sun Sep 30 14:03:32 2007 New Revision: 47039 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: Remove select, add _demo Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Sun Sep 30 14:03:32 2007 @@ -35,13 +35,13 @@ - rewrite the following rctypes modules using rffi: - bz2 - crypt + - _demo - _ssl - errno - fcntl - mmap - rctime - readline - - select - signal - rewrite or kill the extcompiler From fijal at codespeak.net Sun Sep 30 14:06:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 14:06:17 +0200 (CEST) Subject: [pypy-svn] r47040 - pypy/dist/pypy/doc Message-ID: <20070930120617.77D9680F5@code0.codespeak.net> Author: fijal Date: Sun Sep 30 14:06:16 2007 New Revision: 47040 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: Add rsdl and rcairo, remove errno as it does not use rctypes. Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Sun Sep 30 14:06:16 2007 @@ -37,11 +37,12 @@ - crypt - _demo - _ssl - - errno - fcntl - mmap - rctime - readline - signal + - rsdl + - rcairo - rewrite or kill the extcompiler From fijal at codespeak.net Sun Sep 30 15:53:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 15:53:45 +0200 (CEST) Subject: [pypy-svn] r47041 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20070930135345.42DF780E6@code0.codespeak.net> Author: fijal Date: Sun Sep 30 15:53:42 2007 New Revision: 47041 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: update accomodation Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sun Sep 30 15:53:42 2007 @@ -14,7 +14,7 @@ Jacob Hallen lives there Laura Creighton lives there Christian Tismer Laura and Jacob's -Maciej Fijalkowski ? ? +Maciej Fijalkowski ? Laura and Jacob's ==================== ============== ===================== From cfbolz at codespeak.net Sun Sep 30 18:16:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 30 Sep 2007 18:16:55 +0200 (CEST) Subject: [pypy-svn] r47043 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20070930161655.6745080E2@code0.codespeak.net> Author: cfbolz Date: Sun Sep 30 18:16:53 2007 New Revision: 47043 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: failing test: an external function allocates something and returns it. When freeing this after conversion back to an lltype structure there is a failing assertion. Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Sep 30 18:16:53 2007 @@ -648,3 +648,20 @@ assert abs(float(b[1]) - 1.1) < 1E-6 assert isinstance(b[2], rffi.r_singlefloat) assert abs(float(b[2]) - 2.2) < 1E-6 + + def test_cfunc_returning_newly_allocated(self): + py.test.skip("complains about a double free") + from crypt import crypt as pycrypt + crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], + rffi.CCHARP, + libraries=['crypt']) + + s1 = rffi.str2charp("pass") + s2 = rffi.str2charp("ab") + r = crypt(s1, s2) + rffi.free_charp(s1) + rffi.free_charp(s2) + res = rffi.charp2str(r) + assert res == pycrypt("pass", "ab") + rffi.free_charp(r) + assert not ALLOCATED From fijal at codespeak.net Sun Sep 30 21:32:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 21:32:10 +0200 (CEST) Subject: [pypy-svn] r47045 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20070930193210.DE85A8111@code0.codespeak.net> Author: fijal Date: Sun Sep 30 21:32:09 2007 New Revision: 47045 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: I think these frees are good. but ALLOCATED logic complains about that. Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Sep 30 21:32:09 2007 @@ -239,6 +239,7 @@ assert res[3] == '\x00' # XXX maybe we should also allow res[-1], res[-2]... rffi.free_charp(s) + lltype.free(res, flavor='raw') assert not ALLOCATED # detects memory leaks in the test def test_frexp(self): @@ -613,6 +614,7 @@ p = _rsocket_rffi.inet_ntoa(buf) assert rffi.charp2str(p) == '1.2.3.4' lltype.free(buf, flavor='raw') + lltype.free(p, flavor='raw') assert not ALLOCATED # detects memory leaks in the test def test_storage_stays_around(self): @@ -650,7 +652,6 @@ assert abs(float(b[2]) - 2.2) < 1E-6 def test_cfunc_returning_newly_allocated(self): - py.test.skip("complains about a double free") from crypt import crypt as pycrypt crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, From fijal at codespeak.net Sun Sep 30 21:41:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 Sep 2007 21:41:24 +0200 (CEST) Subject: [pypy-svn] r47047 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20070930194124.211138111@code0.codespeak.net> Author: fijal Date: Sun Sep 30 21:41:23 2007 New Revision: 47047 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Revert previous revision Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Sep 30 21:41:23 2007 @@ -239,7 +239,6 @@ assert res[3] == '\x00' # XXX maybe we should also allow res[-1], res[-2]... rffi.free_charp(s) - lltype.free(res, flavor='raw') assert not ALLOCATED # detects memory leaks in the test def test_frexp(self): @@ -614,7 +613,6 @@ p = _rsocket_rffi.inet_ntoa(buf) assert rffi.charp2str(p) == '1.2.3.4' lltype.free(buf, flavor='raw') - lltype.free(p, flavor='raw') assert not ALLOCATED # detects memory leaks in the test def test_storage_stays_around(self): @@ -652,6 +650,7 @@ assert abs(float(b[2]) - 2.2) < 1E-6 def test_cfunc_returning_newly_allocated(self): + py.test.skip("complains about a double free") from crypt import crypt as pycrypt crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, From arigo at codespeak.net Sun Sep 30 22:14:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Sep 2007 22:14:31 +0200 (CEST) Subject: [pypy-svn] r47049 - in pypy/dist/pypy: module/thread rpython/lltypesystem Message-ID: <20070930201431.51BAC8112@code0.codespeak.net> Author: arigo Date: Sun Sep 30 22:14:29 2007 New Revision: 47049 Modified: pypy/dist/pypy/module/thread/ll_thread.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Make an explicit flag 'threadsafe' to know if we can release the GIL around an external function call or not. The default is normally True but see also the comment. Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Sun Sep 30 22:14:29 2007 @@ -47,13 +47,13 @@ # GIL to be released. To use to handle the GIL lock itself. c_thread_acquirelock_NOAUTO = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT, - sandboxsafe=True) + threadsafe=False) c_thread_releaselock_NOAUTO = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void, - sandboxsafe=True) + threadsafe=False) c_thread_fused_releaseacquirelock_NOAUTO = llexternal( 'RPyThreadFusedReleaseAcquireLock', [TLOCKP], lltype.Void, - sandboxsafe=True) + threadsafe=False) def allocate_lock(): ll_lock = lltype.malloc(TLOCKP.TO, flavor='raw') Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Sun Sep 30 22:14:29 2007 @@ -28,7 +28,8 @@ def llexternal(name, args, result, _callable=None, sources=[], includes=[], libraries=[], include_dirs=[], sandboxsafe=False, - canraise=False, _nowrapper=False, calling_conv='c'): + canraise=False, _nowrapper=False, calling_conv='c', + threadsafe='auto'): """Build an external function that will invoke the C function 'name' with the given 'args' types and 'result' type. @@ -37,6 +38,12 @@ CCHARP argument is expected, and the C function receives a 'const char*' pointing to a read-only null-terminated character of arrays, as usual for C. + + threadsafe: whether it's ok to release the GIL around the call. + Default is yes, unless sandboxsafe is set, in which case + we consider that the function is really short-running and + don't bother releasing the GIL. An explicit True or False + overrides this logic. """ ext_type = lltype.FuncType(args, result) if _callable is None: @@ -56,7 +63,16 @@ if _nowrapper: return funcptr - invoke_around_handlers = not sandboxsafe + if threadsafe in (False, True): + # invoke the around-handlers, which release the GIL, if and only if + # the C function is thread-safe. + invoke_around_handlers = threadsafe + else: + # default case: + # invoke the around-handlers only for "not too small" external calls; + # sandboxsafe is a hint for "too-small-ness" (e.g. math functions). + invoke_around_handlers = not sandboxsafe + unrolling_arg_tps = unrolling_iterable(enumerate(args)) def wrapper(*args): # XXX the next line is a workaround for the annotation bug