SMP using Shared memory (was: Re: Is Stackless and uthread.py SMP friendly?)

[Johannes Nix]

jbalding at hotmail.com writes:

> Okay. My next question: Is there a simple way of taking advantage of
> multiple processors using Python, or is this something that is going to
> be like pulling teeth? In other words, are there existing packages that
> allow relatively high-level, transparent access to multiple processors?
> 

I had the same problem; I needed to compute parellel using Numerical
Python on a dual processor workstation.

My solution was to get the shm module (see
http://www.vex.net/parnassus/apyllo.py/990075885.17269372 or
http://sirac.inrialpes.fr/~marangoz/python/shm/), and to write two
small wrapper classes around it to transfer arrays.

Then, I start up my process, do all the initialization, do an
os.fork(), and start passing arrays via shared memory. If you want to
use tkinter, you need to initialize it _after_ os.fork(), or strange
things will happen.


This is nearly as easy as using threads, but works fine, and seems a
lot easier than using PVM. It is also much faster, since shared memory
means that two processes share the same adress space, and no kernel or
networking code needs to run for transfers.

Some remarks about using shared memory on Linux: Note, to compile
shmmodule.c on linux, you need the patch below.  Note also, that the
keys/IDs need to have values larger than zero.

Finally note, the resource limits on Linux can be queried using the
command "ipcs -l", and you can change system parameters in
/usr/src/linux/include/asm/shmparam.h and
/usr/src/linux/include/linux/sem.h (thanks to Detlef Antoni for
pointing this out).


Personally, I would like it very much if Python would get rid of the
global lock, and introduces something like object locks. I think for a
number of people this would be a good reason to switch from Matlab to
Python, or at least give it a try.

and believe me, parallel crunching with Python is FUN!

Johannes


=======================================================================
# file: shmbuf.py
#!/usr/bin/python

""" this module provides a shared memory buffer for arbitrary strings.
It can be used to exchange data between different Python processes,
to overcome the problem with the Global Interpreter Lock
on SMP machines."""

import shm
import sys
from struct import calcsize, pack, unpack

WLOCK='wlock'
RLOCK='rlock'
MUTEX='mutex'

class SHMBuf:
    def __init__(self, shm_key, sem_mutex_key, sem_wlock_key, sem_rlock_key, msize):
        if not (shm_key and sem_mutex_key and sem_wlock_key and sem_rlock_key):
            raise "zero keys do not work."
        self.sems={}
        for tag, semkey, initval in [ (MUTEX,sem_mutex_key,1),
                                      (WLOCK, sem_wlock_key,1),
                                      (RLOCK, sem_rlock_key,0)]:
            try:
                if shm.semaphore_haskey(semkey):
                    s = shm.semaphore(shm.getsemid(semkey))
                    s.setval(initval)
                else:
                    s = shm.create_semaphore(semkey,initval)
            except:
                print "can't acces / allocate semaphore with key %s" % semkey
                raise
            self.sems[tag]=s
        try:
            if shm.memory_haskey(shm_key):
                self.sharedmem = shm.memory(shm.getshmid(shm_key))
            else:
                self.sharedmem = shm.create_memory(shm_key,msize)
        except:
            print "can't access / allocate shared memory with key %s" % shm_key
            raise
        self.sharedmem.attach()

        self.size_long = calcsize('l')

    def write(self,string):
        self.sems[WLOCK].P()
        self.sems[MUTEX].P()
        self.sharedmem.write(pack('l',len(string))+string)
        self.sems[MUTEX].V()
        self.sems[RLOCK].V()

    def read(self):
        self.sems[RLOCK].P()
        self.sems[MUTEX].P()
        size = unpack('l',self.sharedmem.read(self.size_long))[0]
        string = self.sharedmem.read(size,self.size_long)
        self.sems[MUTEX].V()
        self.sems[WLOCK].V()
        return string
===========================================================================
# File: shmarray.py
#!/usr/bin/python


"""This module provides exchange of Numerical array instances
between independent Python processes.

Array shape has to be known, because all arrays come out flattened
(no problem to fix this, but we want it very efficient and it isn't
needed)"""


from shmbuf import *
from Numeric import fromstring

class SHMArray(SHMBuf):
    def write_array(self,array):
        SHMBuf.write(self,array.typecode() + array.tostring())


    def read_array(self):
        s = SHMBuf.read(self)
        return fromstring(s[1:],s[0])



##################################################################

if __name__ == "__main__":
    print "testing..."
    smar = SHMArray(1,1,2,3,1024)

    import os
    import sys
    import time
    from Numeric import arange

    if len(sys.argv) > 1:
        print sys.argv[1]
        tty = open(sys.argv[1],'w')
        print "hello"
        tty.write('Hi there\n')
    else:
        tty = sys.stdout


    child_pid=os.fork()


    if not child_pid:
        # child path
        
        a = arange(20)
        try:
            for i in range(1000):
                smar.write_array(a)
		a  = a * 1.1

        finally:
            print 'child: ready, terminating'
            sys.exit(0)       



    else:
        # this is the parent process
        print "child is:",child_pid
        for i in range(1000):
            tty.write(repr(smar.read_array())+'\n')
        tty.write('parent: ready, terminating\n\n')
        time.sleep(0.05)          # this is for safety, in case
        os.kill(child_pid,9)   # ... child is still waiting.
        sys.exit(0)
            

            
========================================================================
/* patch for shmmodule.py */


--- shm.orig/shmmodule.c	Wed Nov 22 20:12:25 2000
+++ shm/shmmodule.c	Fri Dec 22 16:21:22 2000
@@ -173,6 +173,11 @@
     struct semid_ds ds;		/* data structure	*/
 } PyShmSemaphoreObject;
 
+#ifdef _SEM_SEMUN_UNDEFINED
+#undef HAVE_UNION_SEMUN
+#endif
+
+
 #ifndef HAVE_UNION_SEMUN
 union semun {
     int val;                    /* used for SETVAL only */
@@ -429,7 +434,7 @@
     PyObject *args;
 {
     char *data;
-    unsigned long n, offset = 0;
+    int n, offset = 0;
     char buf[128];
     char *addr;