[Python-checkins] bpo-32576: use queue.SimpleQueue in critical places (#5216)
Antoine Pitrou
webhook-mailer at python.org
Thu Jan 18 04:38:12 EST 2018
https://github.com/python/cpython/commit/ab74504346a6e2569b3255b7b621c589716888c4
commit: ab74504346a6e2569b3255b7b621c589716888c4
branch: master
author: Antoine Pitrou <pitrou at free.fr>
committer: GitHub <noreply at github.com>
date: 2018-01-18T10:38:03+01:00
summary:
bpo-32576: use queue.SimpleQueue in critical places (#5216)
Where a queue may be invoked from a weakref callback, we need
to use the reentrant SimpleQueue.
files:
A Misc/NEWS.d/next/Library/2018-01-17-13-04-16.bpo-32576.iDL09t.rst
M Lib/concurrent/futures/thread.py
M Lib/multiprocessing/pool.py
diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py
index 2e7100bc352..6e22950a157 100644
--- a/Lib/concurrent/futures/thread.py
+++ b/Lib/concurrent/futures/thread.py
@@ -128,7 +128,7 @@ def __init__(self, max_workers=None, thread_name_prefix='',
raise TypeError("initializer must be a callable")
self._max_workers = max_workers
- self._work_queue = queue.Queue()
+ self._work_queue = queue.SimpleQueue()
self._threads = set()
self._broken = False
self._shutdown = False
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py
index b1ee725fac6..3e9a0d6b486 100644
--- a/Lib/multiprocessing/pool.py
+++ b/Lib/multiprocessing/pool.py
@@ -156,7 +156,7 @@ def __init__(self, processes=None, initializer=None, initargs=(),
maxtasksperchild=None, context=None):
self._ctx = context or get_context()
self._setup_queues()
- self._taskqueue = queue.Queue()
+ self._taskqueue = queue.SimpleQueue()
self._cache = {}
self._state = RUN
self._maxtasksperchild = maxtasksperchild
@@ -802,15 +802,18 @@ def __init__(self, processes=None, initializer=None, initargs=()):
Pool.__init__(self, processes, initializer, initargs)
def _setup_queues(self):
- self._inqueue = queue.Queue()
- self._outqueue = queue.Queue()
+ self._inqueue = queue.SimpleQueue()
+ self._outqueue = queue.SimpleQueue()
self._quick_put = self._inqueue.put
self._quick_get = self._outqueue.get
@staticmethod
def _help_stuff_finish(inqueue, task_handler, size):
- # put sentinels at head of inqueue to make workers finish
- with inqueue.not_empty:
- inqueue.queue.clear()
- inqueue.queue.extend([None] * size)
- inqueue.not_empty.notify_all()
+ # drain inqueue, and put sentinels at its head to make workers finish
+ try:
+ while True:
+ inqueue.get(block=False)
+ except queue.Empty:
+ pass
+ for i in range(size):
+ inqueue.put(None)
diff --git a/Misc/NEWS.d/next/Library/2018-01-17-13-04-16.bpo-32576.iDL09t.rst b/Misc/NEWS.d/next/Library/2018-01-17-13-04-16.bpo-32576.iDL09t.rst
new file mode 100644
index 00000000000..143a83e0fcc
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-01-17-13-04-16.bpo-32576.iDL09t.rst
@@ -0,0 +1,2 @@
+Use queue.SimpleQueue() in places where it can be invoked from a weakref
+callback.
More information about the Python-checkins
mailing list