[pypy-svn] r16260 - in pypy/dist/pypy: annotation module/thread/rpython rpython translator/c translator/c/src translator/c/test

arigo at codespeak.net arigo at codespeak.net
Tue Aug 23 13:49:56 CEST 2005


Author: arigo
Date: Tue Aug 23 13:49:53 2005
New Revision: 16260

Modified:
   pypy/dist/pypy/annotation/bookkeeper.py
   pypy/dist/pypy/module/thread/rpython/exttable.py
   pypy/dist/pypy/module/thread/rpython/ll_thread.py
   pypy/dist/pypy/rpython/rtyper.py
   pypy/dist/pypy/translator/c/extfunc.py
   pypy/dist/pypy/translator/c/src/ll_thread.h
   pypy/dist/pypy/translator/c/src/thread_nt.h
   pypy/dist/pypy/translator/c/src/thread_pthread.h
   pypy/dist/pypy/translator/c/test/test_extfunc.py
   pypy/dist/pypy/translator/c/test/test_genc.py
Log:
(rxe, arigo)

Finished basic thread support in the C back-end.

* lots of hacks to support in the annotator/rtyper the notion of
  a call-back function, i.e. a function passed as argument to a built-in
  and that is implicitely called by the built-in.  See emulate_pbc_call().

* the thread_*.h have PyThreadStartThread() and PyThreadGetIdent(), called
  by ll_thread.h.  (Still copied from CPython, still untested on NT.)

* new test in genc.  Also supports view=True in test_genc.compile().

* a fix(?) in bookkeeper.py.  Must run all the tests to be sure.



Modified: pypy/dist/pypy/annotation/bookkeeper.py
==============================================================================
--- pypy/dist/pypy/annotation/bookkeeper.py	(original)
+++ pypy/dist/pypy/annotation/bookkeeper.py	Tue Aug 23 13:49:53 2005
@@ -192,6 +192,7 @@
         self.pbc_callables = None
         
         self.pbc_call_sites = {}
+        self.emulated_pbc_calls = {}
 
         self.needs_hash_support = {}
 
@@ -228,6 +229,11 @@
             self.consider_pbc_call(pbc, shape, spaceop)
         self.pbc_call_sites = {}
 
+        for fn, shape in self.emulated_pbc_calls.iteritems():
+            pbc = SomePBC({fn: True})
+            self.consider_pbc_call(pbc, shape)
+        self.emulated_pbc_calls = {}
+
         for cls in self.needs_hash_support.keys():
             for cls2 in self.needs_hash_support:
                 if issubclass(cls, cls2) and cls is not cls2:
@@ -475,8 +481,9 @@
         else:
             implicit_init = None
 
-        pbc, dontcaresc = self.query_spaceop_callable(spaceop,
-                                                      implicit_init=implicit_init) 
+        if not (spaceop is implicit_init is None):
+            pbc, dontcaresc = self.query_spaceop_callable(spaceop,
+                                            implicit_init=implicit_init) 
 
         nonnullcallables = []
         for func, classdef in pbc.prebuiltinstances.items():
@@ -521,7 +528,9 @@
                 self.pbc_call_sites[self.position_key] = shape
 
         results = []
-        nonnullcallables = [(func, classdef) for func, classdef in pbc.prebuiltinstances.items()]
+        nonnullcallables = [(func, classdef)
+                            for func, classdef in pbc.prebuiltinstances.items()
+                            if func is not None]
         mono = len(nonnullcallables) == 1
 
         for func, classdef in nonnullcallables:
@@ -534,6 +543,18 @@
 
         return unionof(*results) 
 
+    def emulate_pbc_call(self, pbc, args_s):
+        args = self.build_args("simple_call", args_s)
+        shape = args.rawshape()
+        for func, classdef in pbc.prebuiltinstances.items():
+            if func is not None:
+                assert not isclassdef(classdef)
+                if func in self.emulated_pbc_calls:
+                    assert shape == self.emulated_pbc_calls[func]
+                else:
+                    self.emulated_pbc_calls[func] = shape
+        return self.pbc_call(pbc, args, True)
+
     # decide_callable(position, func, args, mono) -> callb, key
     # query_spaceop_callable(spaceop) -> pbc, isspecialcase
     # get_s_init(decided_cls) -> classdef, s_undecided_init

Modified: pypy/dist/pypy/module/thread/rpython/exttable.py
==============================================================================
--- pypy/dist/pypy/module/thread/rpython/exttable.py	(original)
+++ pypy/dist/pypy/module/thread/rpython/exttable.py	Tue Aug 23 13:49:53 2005
@@ -19,9 +19,24 @@
 # ____________________________________________________________
 # Built-in functions needed in the rtyper
 
-declare(thread.start_new_thread, int,            '%s/start_new_thread' % module)
+def ann_startthr(s_bootstrap_function, s_argument_tuple):
+    from pypy.annotation import model as annmodel
+    from pypy.annotation.bookkeeper import getbookkeeper
+    bookkeeper = getbookkeeper()
+    assert (isinstance(s_argument_tuple, annmodel.SomeTuple) and
+            len(s_argument_tuple.items) == 1), (
+        """thread.start_new_thread(f, arg) is only supported with a tuple of
+           length 1 for arg""")
+    s_arg, = s_argument_tuple.items
+    # XXX hack hack hack: emulate a call to s_bootstrap_function
+    s_result = bookkeeper.emulate_pbc_call(s_bootstrap_function, [s_arg])
+    assert bookkeeper.getpbc(None).contains(s_result), (
+        """thread.start_new_thread(f, arg): f() should return None""")
+    return annmodel.SomeInteger()
+
+declare(thread.start_new_thread, ann_startthr,   '%s/start_new_thread' % module)
 declare(thread.get_ident,        int,            '%s/get_ident'        % module)
-declare(thread.allocate_lock,   thread.LockType, '%s/allocate_lock'    % module)
+declare(thread.allocate_lock,    thread.LockType,'%s/allocate_lock'    % module)
 
 # ____________________________________________________________
 # thread.error can be raised by the above

Modified: pypy/dist/pypy/module/thread/rpython/ll_thread.py
==============================================================================
--- pypy/dist/pypy/module/thread/rpython/ll_thread.py	(original)
+++ pypy/dist/pypy/module/thread/rpython/ll_thread.py	Tue Aug 23 13:49:53 2005
@@ -18,7 +18,14 @@
     return ll_thread_start(funcptr, argument)
 
 def ll_thread_start(funcptr, argument):
-    return thread.start_new_thread(funcptr, (argument,))
+    #return thread.start_new_thread(funcptr, (argument,))
+    # XXX we just return an integer here, because we cannot really call back
+    # XXX into thread.start_new_thread().  Indeed, 'funcptr' is not a normal
+    # XXX function object, but a low-level pointer to a _func.  This also
+    # XXX confuses the annotator.
+    # note that running this won't really work, but anyway llinterpreter
+    # is probably quite confused if we start multiple threads
+    return 1234
 ll_thread_start.suggested_primitive = True
 
 def ll_thread_get_ident():

Modified: pypy/dist/pypy/rpython/rtyper.py
==============================================================================
--- pypy/dist/pypy/rpython/rtyper.py	(original)
+++ pypy/dist/pypy/rpython/rtyper.py	Tue Aug 23 13:49:53 2005
@@ -702,15 +702,25 @@
     def gendirectcall(self, ll_function, *args_v):
         rtyper = self.rtyper
         args_s = []
+        newargs_v = []
         for v in args_v:
             if v.concretetype == Void:
                 s_value = rtyper.binding(v)
                 if not s_value.is_constant():
                     raise TyperError("non-constant variable of type Void")
+                if not isinstance(s_value, annmodel.SomePBC):
+                    # a Void non-PBC constant: can be a SomePtr pointing to a
+                    # constant function.
+                    assert isinstance(s_value, annmodel.SomePtr)
+                    # Drop the 'const'.
+                    s_value = annmodel.SomePtr(s_value.ll_ptrtype)
+                    # Modify args_v so that 'v' gets the llptr concretetype
+                    # stored in s_value
+                    v = inputconst(s_value.ll_ptrtype, v.value)
                 args_s.append(s_value)
-                assert isinstance(s_value, annmodel.SomePBC)
             else:
                 args_s.append(annmodel.lltype_to_annotation(v.concretetype))
+            newargs_v.append(v)
         
         self.rtyper.call_all_setups()  # compute ForwardReferences now
         dontcare, spec_function = annotate_lowlevel_helper(rtyper.annotator, ll_function, args_s)
@@ -718,7 +728,7 @@
         # build the 'direct_call' operation
         f = self.rtyper.getfunctionptr(spec_function)
         c = inputconst(typeOf(f), f)
-        return self.genop('direct_call', [c]+list(args_v),
+        return self.genop('direct_call', [c]+newargs_v,
                           resulttype = typeOf(f).TO.RESULT)
 
     def genexternalcall(self, fnname, args_v, resulttype=None, **flags):

Modified: pypy/dist/pypy/translator/c/extfunc.py
==============================================================================
--- pypy/dist/pypy/translator/c/extfunc.py	(original)
+++ pypy/dist/pypy/translator/c/extfunc.py	Tue Aug 23 13:49:53 2005
@@ -37,9 +37,11 @@
         'LL_strtod_parts_to_float',
     ll_strtod.ll_strtod_formatd:
         'LL_strtod_formatd',
-    ll_thread.ll_newlock:     'LL_thread_newlock',
-    ll_thread.ll_acquirelock: 'LL_thread_acquirelock',
-    ll_thread.ll_releaselock: 'LL_thread_releaselock',
+    ll_thread.ll_newlock:          'LL_thread_newlock',
+    ll_thread.ll_acquirelock:      'LL_thread_acquirelock',
+    ll_thread.ll_releaselock:      'LL_thread_releaselock',
+    ll_thread.ll_thread_start:     'LL_thread_start',
+    ll_thread.ll_thread_get_ident: 'LL_thread_get_ident',
     }
 
 #______________________________________________________

Modified: pypy/dist/pypy/translator/c/src/ll_thread.h
==============================================================================
--- pypy/dist/pypy/translator/c/src/ll_thread.h	(original)
+++ pypy/dist/pypy/translator/c/src/ll_thread.h	Tue Aug 23 13:49:53 2005
@@ -35,3 +35,18 @@
 		RPyThreadReleaseLock(lock);
 	}
 }
+
+long LL_thread_start(void (*func)(void *), void *arg)
+{
+	/* XXX func() should not raise exceptions */
+	long ident = RPyThreadStart(func, arg);
+	if (ident == -1)
+		RPyRaiseSimpleException(PyExc_thread_error,
+					"can't start new thread");
+	return ident;
+}
+
+long LL_thread_get_ident(void)
+{
+	return RPyThreadGetIdent();
+}

Modified: pypy/dist/pypy/translator/c/src/thread_nt.h
==============================================================================
--- pypy/dist/pypy/translator/c/src/thread_nt.h	(original)
+++ pypy/dist/pypy/translator/c/src/thread_nt.h	Tue Aug 23 13:49:53 2005
@@ -8,6 +8,68 @@
 #include <limits.h>
 #include <process.h>
 
+
+/*
+ * Thread support.
+ */
+
+/*
+ * Return the thread Id instead of an handle. The Id is said to uniquely
+   identify the thread in the system
+ */
+#define PyThreadGetIdent GetCurrentThreadId
+
+typedef struct {
+	void (*func)(void*);
+	void *arg;
+	long id;
+	HANDLE done;
+} callobj;
+
+static int
+bootstrap(void *call)
+{
+	callobj *obj = (callobj*)call;
+	/* copy callobj since other thread might free it before we're done */
+	void (*func)(void*) = obj->func;
+	void *arg = obj->arg;
+
+	obj->id = PyThreadGetIdent();
+	ReleaseSemaphore(obj->done, 1, NULL);
+	func(arg);
+	return 0;
+}
+
+long PyThreadStart(void (*func)(void *), void *arg)
+{
+	unsigned long rv;
+	callobj obj;
+
+	obj.id = -1;	/* guilty until proved innocent */
+	obj.func = func;
+	obj.arg = arg;
+	obj.done = CreateSemaphore(NULL, 0, 1, NULL);
+	if (obj.done == NULL)
+		return -1;
+
+	rv = _beginthread(bootstrap, 0, &obj); /* use default stack size */
+	if (rv == (unsigned long)-1) {
+		/* I've seen errno == EAGAIN here, which means "there are
+		 * too many threads".
+		 */
+		obj.id = -1;
+	}
+	else {
+		/* wait for thread to initialize, so we can get its id */
+		WaitForSingleObject(obj.done, INFINITE);
+		assert(obj.id != -1);
+	}
+	CloseHandle((HANDLE)obj.done);
+	return obj.id;
+}
+
+/************************************************************/
+
 typedef struct RPyOpaque_ThreadLock {
 	LONG   owned ;
 	DWORD  thread_id ;

Modified: pypy/dist/pypy/translator/c/src/thread_pthread.h
==============================================================================
--- pypy/dist/pypy/translator/c/src/thread_pthread.h	(original)
+++ pypy/dist/pypy/translator/c/src/thread_pthread.h	Tue Aug 23 13:49:53 2005
@@ -40,6 +40,69 @@
 #define CHECK_STATUS(name)  if (status != 0) { perror(name); error = 1; }
 
 
+/* XXX This implementation is considered (to quote Tim Peters) "inherently
+   hosed" because:
+     - It does not guarantee the promise that a non-zero integer is returned.
+     - The cast to long is inherently unsafe.
+     - It is not clear that the 'volatile' (for AIX?) and ugly casting in the
+       latter return statement (for Alpha OSF/1) are any longer necessary.
+*/
+long RPyThreadGetIdent(void)
+{
+	volatile pthread_t threadid;
+	/* Jump through some hoops for Alpha OSF/1 */
+	threadid = pthread_self();
+#if SIZEOF_PTHREAD_T <= SIZEOF_LONG
+	return (long) threadid;
+#else
+	return (long) *(long *) &threadid;
+#endif
+}
+
+long RPyThreadStart(void (*func)(void *), void *arg)
+{
+	pthread_t th;
+	int status;
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+	pthread_attr_t attrs;
+#endif
+
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+	pthread_attr_init(&attrs);
+#endif
+#ifdef THREAD_STACK_SIZE
+	pthread_attr_setstacksize(&attrs, THREAD_STACK_SIZE);
+#endif
+#if defined(PTHREAD_SYSTEM_SCHED_SUPPORTED) && !defined(__FreeBSD__)
+        pthread_attr_setscope(&attrs, PTHREAD_SCOPE_SYSTEM);
+#endif
+
+	status = pthread_create(&th, 
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+				 &attrs,
+#else
+				 (pthread_attr_t*)NULL,
+#endif
+				 (void* (*)(void *))func,
+				 (void *)arg
+				 );
+
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+	pthread_attr_destroy(&attrs);
+#endif
+	if (status != 0)
+            return -1;
+
+        pthread_detach(th);
+
+#if SIZEOF_PTHREAD_T <= SIZEOF_LONG
+	return (long) th;
+#else
+	return (long) *(long *) &th;
+#endif
+}
+
+
 /************************************************************/
 #ifdef USE_SEMAPHORES
 /************************************************************/

Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_extfunc.py	(original)
+++ pypy/dist/pypy/translator/c/test/test_extfunc.py	Tue Aug 23 13:49:53 2005
@@ -321,3 +321,54 @@
     f = compile(fn, [])
     res = f()
     assert res is True
+
+def test_simple_start_new_thread():
+    import thread
+    import pypy.module.thread.rpython.exttable   # for declare()/declaretype()
+    class Arg:
+        pass
+    def mythreadedfunction(arg):
+        assert arg.value == 42
+    def myotherthreadedfunction(arg):
+        assert arg.value == 43
+    def fn(i):
+        a42 = Arg()
+        a42.value = 42
+        a43 = Arg()
+        a43.value = 43
+        thread.start_new_thread(mythreadedfunction, (a42,))
+        thread.start_new_thread(myotherthreadedfunction, (a43,))
+        if i == 1:
+            x = mythreadedfunction
+            a = a42
+        else:
+            x = myotherthreadedfunction
+            a = a43
+        thread.start_new_thread(x, (a,))
+        return 42
+    f = compile(fn, [int])
+    res = f(1)
+    assert res == 42
+
+def test_start_new_thread():
+    import thread
+    import pypy.module.thread.rpython.exttable   # for declare()/declaretype()
+    class Arg:
+        pass
+    def mythreadedfunction(arg):
+        arg.x += 37
+        arg.myident = thread.get_ident()
+        arg.lock.release()
+    def fn():
+        a = Arg()
+        a.x = 5
+        a.lock = thread.allocate_lock()
+        a.lock.acquire(True)
+        ident = thread.start_new_thread(mythreadedfunction, (a,))
+        assert ident != thread.get_ident()
+        a.lock.acquire(True)  # wait for the thread to finish
+        assert a.myident == ident
+        return a.x
+    f = compile(fn, [])
+    res = f()
+    assert res == 42

Modified: pypy/dist/pypy/translator/c/test/test_genc.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_genc.py	(original)
+++ pypy/dist/pypy/translator/c/test/test_genc.py	Tue Aug 23 13:49:53 2005
@@ -26,10 +26,12 @@
                            include_dirs = [os.path.dirname(autopath.this_dir)])
     return m
 
-def compile(fn, argtypes):
+def compile(fn, argtypes, view=False):
     t = Translator(fn)
     t.annotate(argtypes)
     t.specialize()
+    if view:
+        t.view()
     t.backend_optimizations()
     db = LowLevelDatabase(t)
     entrypoint = db.get(pyobjectptr(fn))
@@ -209,3 +211,16 @@
     f1 = compile(fn, [])
     res = f1()
     assert res > 0 and res == res / 2
+
+
+def test_x():
+    class A:
+        pass
+    a = A()
+    a.d = {}
+    a.d['hey'] = 42
+    def t():
+        a.d['hey'] = 2
+        return a.d['hey']
+    f = compile(t, [])
+    assert f() == 2



More information about the Pypy-commit mailing list