[Python-checkins] bpo-35615: Fix crashes when copying a Weak{Key, Value}Dictionary. (GH-11384)
Antoine Pitrou
webhook-mailer at python.org
Thu Feb 7 14:52:05 EST 2019
https://github.com/python/cpython/commit/96d37dbcd23e65a7a57819aeced9034296ef747e
commit: 96d37dbcd23e65a7a57819aeced9034296ef747e
branch: master
author: Fish <ltfish at users.noreply.github.com>
committer: Antoine Pitrou <pitrou at free.fr>
date: 2019-02-07T19:51:59Z
summary:
bpo-35615: Fix crashes when copying a Weak{Key,Value}Dictionary. (GH-11384)
Protect dict iterations by wrapping them with _IterationGuard in the
following methods:
- WeakValueDictionary.copy()
- WeakValueDictionary.__deepcopy__()
- WeakKeyDictionary.copy()
- WeakKeyDictionary.__deepcopy__()
files:
A Misc/NEWS.d/next/Library/2018-12-30-20-00-05.bpo-35615.Uz1SVh.rst
M Lib/test/test_weakref.py
M Lib/weakref.py
diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py
index 9fa0bbd78087..1fac08dafc7d 100644
--- a/Lib/test/test_weakref.py
+++ b/Lib/test/test_weakref.py
@@ -8,6 +8,7 @@
import copy
import threading
import time
+import random
from test import support
from test.support import script_helper
@@ -1688,6 +1689,87 @@ def test_threaded_weak_valued_consistency(self):
self.assertEqual(len(d), 1)
o = None # lose ref
+ def check_threaded_weak_dict_copy(self, type_, deepcopy):
+ # `type_` should be either WeakKeyDictionary or WeakValueDictionary.
+ # `deepcopy` should be either True or False.
+ exc = []
+
+ class DummyKey:
+ def __init__(self, ctr):
+ self.ctr = ctr
+
+ class DummyValue:
+ def __init__(self, ctr):
+ self.ctr = ctr
+
+ def dict_copy(d, exc):
+ try:
+ if deepcopy is True:
+ _ = copy.deepcopy(d)
+ else:
+ _ = d.copy()
+ except Exception as ex:
+ exc.append(ex)
+
+ def pop_and_collect(lst):
+ gc_ctr = 0
+ while lst:
+ i = random.randint(0, len(lst) - 1)
+ gc_ctr += 1
+ lst.pop(i)
+ if gc_ctr % 10000 == 0:
+ gc.collect() # just in case
+
+ self.assertIn(type_, (weakref.WeakKeyDictionary, weakref.WeakValueDictionary))
+
+ d = type_()
+ keys = []
+ values = []
+ # Initialize d with many entries
+ for i in range(70000):
+ k, v = DummyKey(i), DummyValue(i)
+ keys.append(k)
+ values.append(v)
+ d[k] = v
+ del k
+ del v
+
+ t_copy = threading.Thread(target=dict_copy, args=(d, exc,))
+ if type_ is weakref.WeakKeyDictionary:
+ t_collect = threading.Thread(target=pop_and_collect, args=(keys,))
+ else: # weakref.WeakValueDictionary
+ t_collect = threading.Thread(target=pop_and_collect, args=(values,))
+
+ t_copy.start()
+ t_collect.start()
+
+ t_copy.join()
+ t_collect.join()
+
+ # Test exceptions
+ if exc:
+ raise exc[0]
+
+ def test_threaded_weak_key_dict_copy(self):
+ # Issue #35615: Weakref keys or values getting GC'ed during dict
+ # copying should not result in a crash.
+ self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False)
+
+ def test_threaded_weak_key_dict_deepcopy(self):
+ # Issue #35615: Weakref keys or values getting GC'ed during dict
+ # copying should not result in a crash.
+ self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True)
+
+ def test_threaded_weak_value_dict_copy(self):
+ # Issue #35615: Weakref keys or values getting GC'ed during dict
+ # copying should not result in a crash.
+ self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False)
+
+ def test_threaded_weak_value_dict_deepcopy(self):
+ # Issue #35615: Weakref keys or values getting GC'ed during dict
+ # copying should not result in a crash.
+ self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, True)
+
from test import mapping_tests
diff --git a/Lib/weakref.py b/Lib/weakref.py
index 99de2eab7414..753f07291e20 100644
--- a/Lib/weakref.py
+++ b/Lib/weakref.py
@@ -171,10 +171,11 @@ def copy(self):
if self._pending_removals:
self._commit_removals()
new = WeakValueDictionary()
- for key, wr in self.data.items():
- o = wr()
- if o is not None:
- new[key] = o
+ with _IterationGuard(self):
+ for key, wr in self.data.items():
+ o = wr()
+ if o is not None:
+ new[key] = o
return new
__copy__ = copy
@@ -184,10 +185,11 @@ def __deepcopy__(self, memo):
if self._pending_removals:
self._commit_removals()
new = self.__class__()
- for key, wr in self.data.items():
- o = wr()
- if o is not None:
- new[deepcopy(key, memo)] = o
+ with _IterationGuard(self):
+ for key, wr in self.data.items():
+ o = wr()
+ if o is not None:
+ new[deepcopy(key, memo)] = o
return new
def get(self, key, default=None):
@@ -408,10 +410,11 @@ def __setitem__(self, key, value):
def copy(self):
new = WeakKeyDictionary()
- for key, value in self.data.items():
- o = key()
- if o is not None:
- new[o] = value
+ with _IterationGuard(self):
+ for key, value in self.data.items():
+ o = key()
+ if o is not None:
+ new[o] = value
return new
__copy__ = copy
@@ -419,10 +422,11 @@ def copy(self):
def __deepcopy__(self, memo):
from copy import deepcopy
new = self.__class__()
- for key, value in self.data.items():
- o = key()
- if o is not None:
- new[o] = deepcopy(value, memo)
+ with _IterationGuard(self):
+ for key, value in self.data.items():
+ o = key()
+ if o is not None:
+ new[o] = deepcopy(value, memo)
return new
def get(self, key, default=None):
diff --git a/Misc/NEWS.d/next/Library/2018-12-30-20-00-05.bpo-35615.Uz1SVh.rst b/Misc/NEWS.d/next/Library/2018-12-30-20-00-05.bpo-35615.Uz1SVh.rst
new file mode 100644
index 000000000000..4aff8f7f30c8
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-12-30-20-00-05.bpo-35615.Uz1SVh.rst
@@ -0,0 +1,3 @@
+:mod:`weakref`: Fix a RuntimeError when copying a WeakKeyDictionary or a
+WeakValueDictionary, due to some keys or values disappearing while
+iterating.
More information about the Python-checkins
mailing list