[Python-ideas] multiprocessing IPC

[shibturn]

On 12/02/2012 2:35pm, Sturla Molden wrote:
> Mark did not use shm_open, he memory mapped from disk.

But if his /tmp is a tmpfs file system (which it usually is on Linux) 
then I think it is entirely equivalent.  Or he could create the file in 
/dev/shm instead.

Below is Blob class which seems to work.  Note that the process which 
created the blob needs to wait for the other process to unpickle it 
before allowing it to be garbage collected.


import multiprocessing as mp
from multiprocessing.util import Finalize, get_temp_dir
import mmap, sys, os, itertools

class Blob(object):

     _counter = itertools.count()

     def __init__(self, length, name=None):
         self.length = length
         if sys.platform == 'win32':
             if name is None:
                 name = 'blob-%s-%d' % (os.getpid(), next(self._counter))
             self.name = name
             self.mmap = mmap.mmap(-1, length, self.name)
         else:
             if name is None:
                 self.name = '%s/blob-%s-%d' % (get_temp_dir(),
                                                os.getpid(),
                                                next(self._counter))
                 flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
             else:
                 self.name = name
                 flags = os.O_RDWR
             fd = os.open(self.name, flags, 0o600)
             try:
                 if name is None:
                     os.ftruncate(fd, length)
                     Finalize(self, os.unlink, (self.name,),
                              exitpriority=0)
                 self.mmap = mmap.mmap(fd, length)
             finally:
                 os.close(fd)

     def __reduce__(self):
         return Blob, (self.length, self.name)

def child(conn):
     b = Blob(20)
     b.mmap[:5] = "hello"
     conn.send(b)
     conn.recv()                 # wait for acknowledgement before
                                 # allowing garbage collection

if __name__ == '__main__':
     conn, child_conn = mp.Pipe()
     p = mp.Process(target=child, args=(child_conn,))
     p.start()
     b = conn.recv()
     conn.send(None)             # acknowledge receipt
     print repr(b.mmap[:])