[Python-checkins] bpo-39850: Add support for abstract sockets in multiprocessing (GH-18866)
Pablo Galindo
webhook-mailer at python.org
Mon Mar 9 09:48:07 EDT 2020
https://github.com/python/cpython/commit/6012f30beff7fa8396718dfb198ccafc333c565b
commit: 6012f30beff7fa8396718dfb198ccafc333c565b
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2020-03-09T13:48:01Z
summary:
bpo-39850: Add support for abstract sockets in multiprocessing (GH-18866)
files:
A Misc/NEWS.d/next/Library/2020-03-09-01-45-06.bpo-39850.eaJNIE.rst
M Lib/multiprocessing/connection.py
M Lib/multiprocessing/forkserver.py
M Lib/multiprocessing/managers.py
M Lib/multiprocessing/util.py
M Lib/test/_test_multiprocessing.py
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index c9f995e5fa7f5..510e4b5aba44a 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -73,6 +73,11 @@ def arbitrary_address(family):
if family == 'AF_INET':
return ('localhost', 0)
elif family == 'AF_UNIX':
+ # Prefer abstract sockets if possible to avoid problems with the address
+ # size. When coding portable applications, some implementations have
+ # sun_path as short as 92 bytes in the sockaddr_un struct.
+ if util.abstract_sockets_supported:
+ return f"\0listener-{os.getpid()}-{next(_mmap_counter)}"
return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
elif family == 'AF_PIPE':
return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
@@ -102,7 +107,7 @@ def address_type(address):
return 'AF_INET'
elif type(address) is str and address.startswith('\\\\'):
return 'AF_PIPE'
- elif type(address) is str:
+ elif type(address) is str or util.is_abstract_socket_namespace(address):
return 'AF_UNIX'
else:
raise ValueError('address type of %r unrecognized' % address)
@@ -597,7 +602,8 @@ def __init__(self, address, family, backlog=1):
self._family = family
self._last_accepted = None
- if family == 'AF_UNIX':
+ if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address):
+ # Linux abstract socket namespaces do not need to be explicitly unlinked
self._unlink = util.Finalize(
self, os.unlink, args=(address,), exitpriority=0
)
diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py
index 87ebef6588acd..215ac39afcaa0 100644
--- a/Lib/multiprocessing/forkserver.py
+++ b/Lib/multiprocessing/forkserver.py
@@ -55,7 +55,8 @@ def _stop_unlocked(self):
os.waitpid(self._forkserver_pid, 0)
self._forkserver_pid = None
- os.unlink(self._forkserver_address)
+ if not util.is_abstract_socket_namespace(self._forkserver_address):
+ os.unlink(self._forkserver_address)
self._forkserver_address = None
def set_forkserver_preload(self, modules_names):
@@ -135,7 +136,8 @@ def ensure_running(self):
with socket.socket(socket.AF_UNIX) as listener:
address = connection.arbitrary_address('AF_UNIX')
listener.bind(address)
- os.chmod(address, 0o600)
+ if not util.is_abstract_socket_namespace(address):
+ os.chmod(address, 0o600)
listener.listen()
# all client processes own the write end of the "alive" pipe;
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index 1f9c2daa25d97..1668220c09957 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -1262,8 +1262,12 @@ class SharedMemoryServer(Server):
def __init__(self, *args, **kwargs):
Server.__init__(self, *args, **kwargs)
+ address = self.address
+ # The address of Linux abstract namespaces can be bytes
+ if isinstance(address, bytes):
+ address = os.fsdecode(address)
self.shared_memory_context = \
- _SharedMemoryTracker(f"shmm_{self.address}_{getpid()}")
+ _SharedMemoryTracker(f"shm_{address}_{getpid()}")
util.debug(f"SharedMemoryServer started by pid {getpid()}")
def create(self, c, typeid, /, *args, **kwargs):
diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py
index 4bc7782c00c15..32c7a96d2534d 100644
--- a/Lib/multiprocessing/util.py
+++ b/Lib/multiprocessing/util.py
@@ -102,6 +102,29 @@ def log_to_stderr(level=None):
_log_to_stderr = True
return _logger
+
+# Abstract socket support
+
+def _platform_supports_abstract_sockets():
+ if sys.platform == "linux":
+ return True
+ if hasattr(sys, 'getandroidapilevel'):
+ return True
+ return False
+
+
+def is_abstract_socket_namespace(address):
+ if not address:
+ return False
+ if isinstance(address, bytes):
+ return address[0] == 0
+ elif isinstance(address, str):
+ return address[0] == "\0"
+ raise TypeError('address type of {address!r} unrecognized')
+
+
+abstract_sockets_supported = _platform_supports_abstract_sockets()
+
#
# Function returning a temp directory which will be removed on exit
#
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 73dc75d34a6f0..b985d51508cb7 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3274,6 +3274,19 @@ def test_context(self):
if self.TYPE == 'processes':
self.assertRaises(OSError, l.accept)
+ @unittest.skipUnless(util.abstract_sockets_supported,
+ "test needs abstract socket support")
+ def test_abstract_socket(self):
+ with self.connection.Listener("\0something") as listener:
+ with self.connection.Client(listener.address) as client:
+ with listener.accept() as d:
+ client.send(1729)
+ self.assertEqual(d.recv(), 1729)
+
+ if self.TYPE == 'processes':
+ self.assertRaises(OSError, listener.accept)
+
+
class _TestListenerClient(BaseTestCase):
ALLOWED_TYPES = ('processes', 'threads')
diff --git a/Misc/NEWS.d/next/Library/2020-03-09-01-45-06.bpo-39850.eaJNIE.rst b/Misc/NEWS.d/next/Library/2020-03-09-01-45-06.bpo-39850.eaJNIE.rst
new file mode 100644
index 0000000000000..bb00dd8eb4b2e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-03-09-01-45-06.bpo-39850.eaJNIE.rst
@@ -0,0 +1,5 @@
+:mod:`multiprocessing` now supports abstract socket addresses (if abstract sockets
+are supported in the running platform). When creating arbitrary addresses (like when
+default-constructing :class:`multiprocessing.connection.Listener` objects) abstract
+sockets are preferred to avoid the case when the temporary-file-generated address is
+too large for an AF_UNIX socket address. Patch by Pablo Galindo.
More information about the Python-checkins
mailing list