[Python-ideas] multiprocessing IPC

shibturn shibturn at gmail.com
Sun Feb 12 16:20:11 CET 2012

On 12/02/2012 2:35pm, Sturla Molden wrote:
> Mark did not use shm_open, he memory mapped from disk.

But if his /tmp is a tmpfs file system (which it usually is on Linux) 
then I think it is entirely equivalent.  Or he could create the file in 
/dev/shm instead.

Below is Blob class which seems to work.  Note that the process which 
created the blob needs to wait for the other process to unpickle it 
before allowing it to be garbage collected.

import multiprocessing as mp
from multiprocessing.util import Finalize, get_temp_dir
import mmap, sys, os, itertools

class Blob(object):

     _counter = itertools.count()

     def __init__(self, length, name=None):
         self.length = length
         if sys.platform == 'win32':
             if name is None:
                 name = 'blob-%s-%d' % (os.getpid(), next(self._counter))
             self.name = name
             self.mmap = mmap.mmap(-1, length, self.name)
             if name is None:
                 self.name = '%s/blob-%s-%d' % (get_temp_dir(),
                 flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
                 self.name = name
                 flags = os.O_RDWR
             fd = os.open(self.name, flags, 0o600)
                 if name is None:
                     os.ftruncate(fd, length)
                     Finalize(self, os.unlink, (self.name,),
                 self.mmap = mmap.mmap(fd, length)

     def __reduce__(self):
         return Blob, (self.length, self.name)

def child(conn):
     b = Blob(20)
     b.mmap[:5] = "hello"
     conn.recv()                 # wait for acknowledgement before
                                 # allowing garbage collection

if __name__ == '__main__':
     conn, child_conn = mp.Pipe()
     p = mp.Process(target=child, args=(child_conn,))
     b = conn.recv()
     conn.send(None)             # acknowledge receipt
     print repr(b.mmap[:])

More information about the Python-ideas mailing list