[concurrency] Adding shm_open to mmap

shibturn shibturn at gmail.com
Wed Feb 15 16:52:38 CET 2012


On Tue, 14 Feb 2012 23:10:11 -0500
Mike Meyer<mwm-tkOQc4lHIczYtjvyW6yDsg at public.gmane.org>  wrote:

> I'd prefer to provide shm_open on Windows if at all possible. The
> "sorta-kinda" bothers me. That would also allow for an application to
> exit and then resume work stored in a mapped segment (something I've
> done before). However, setting this up on Windows isn't something I
> can do.

Here is a proof-of-concept for shm_open/shm_unlink on Windows.

Note that shm_unlink opens the file using FILE_FLAG_DELETE_ON_CLOSE 
which ensures that subsequent attempts to open the file will not 
succeed.  However, the directory entry will not disappear till all 
handles have been closed.

FILE_ATTRIBUTE_TEMPORARY tells the system we want to try to cache the 
file without worrying about flushing.

sbt


import os
import msvcrt
import tempfile
from _multiprocessing import win32

DEV_SHM = tempfile.gettempdir()

GENERIC_READ = win32.GENERIC_READ
GENERIC_WRITE = win32.GENERIC_WRITE

CREATE_NEW = 1
CREATE_ALWAYS = 2
OPEN_EXISTING = 3
OPEN_ALWAYS = 4
TRUNCATE_EXISTING = 5

FILE_SHARE_READ = 1
FILE_SHARE_WRITE = 2
FILE_SHARE_DELETE = 4

FILE_ATTRIBUTE_TEMPORARY = 256
FILE_FLAG_DELETE_ON_CLOSE = 0x04000000

NULL = 0

_CreationDisposition = {
     os.O_CREAT | os.O_EXCL:     CREATE_NEW,
     os.O_CREAT | os.O_TRUNC:    CREATE_ALWAYS,
     0:                          OPEN_EXISTING,
     os.O_CREAT:                 OPEN_ALWAYS,
     os.O_TRUNC:                 TRUNCATE_EXISTING
}

_DesiredAccess = {
     os.O_RDONLY:                GENERIC_READ,
     os.O_RDWR:                  GENERIC_READ | GENERIC_WRITE
}

_OsfDesiredAccess = {
     os.O_RDONLY:                os.O_RDONLY,
     os.O_RDWR:                  0
}

def _get_path(name, dev_shm):
     name = name.lstrip('/')
     if '/' in name or '\\' in name:
         raise ValueError('invalid name')
     return os.path.join(dev_shm, name)

def shm_open(name, flags, mode, dev_shm=DEV_SHM):
     # Opening with FILE_SHARE_READ | FILE_SHARE_WRITE |
     # FILE_SHARE_DELETE ensures that other processes can open the file
     # and pseudo-unlink it while it is still in use.
     path = _get_path(name, dev_shm)
     da_flags = flags & (os.O_RDONLY | os.O_RDWR)
     cd_flags = flags & ~(os.O_RDONLY | os.O_RDWR)
     h = win32.CreateFile(
         path, _DesiredAccess[da_flags],
         FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL,
         _CreationDisposition[cd_flags], FILE_ATTRIBUTE_TEMPORARY, NULL)
     try:
         os.chmod(path, mode)
         return msvcrt.open_osfhandle(h, _OsfDesiredAccess[da_flags])
     except:
         win32.CloseHandle(h)
         raise

def shm_unlink(name, dev_shm=DEV_SHM):
     # Opening with FILE_FLAG_DELETE_ON_CLOSE will make subsequent
     # attempts to open the file fail.  However, the name is not
     # removed from the file system until all handles have been removed.
     path = _get_path(name, dev_shm)
     h = win32.CreateFile(
         path, GENERIC_READ | GENERIC_WRITE,
         FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL,
         OPEN_EXISTING,
         FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE, NULL)
     win32.CloseHandle(h)

##

if __name__ == '__main__':
     import mmap

     MYMAP = "mymmap"

     fd = shm_open(MYMAP, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
     m = mmap.mmap(fd, 10)
     os.close(fd)

     m[:5] = "hello"

     fd = shm_open(MYMAP, os.O_RDWR, 0o600)
     n = mmap.mmap(fd, 10)
     os.close(fd)

     n[:] = n[:].upper()
     n.close()

     assert os.path.exists(_get_path(MYMAP, DEV_SHM))
     shm_unlink(MYMAP)

     try:
         fd = shm_open(MYMAP, os.O_RDWR, 0o600)
     except OSError:
         pass
     else:
         raise AssertionError("expected access denied")

     print repr(m[:])
     m.close()
     assert not os.path.exists(_get_path(MYMAP, DEV_SHM))



More information about the concurrency-sig mailing list