Mailman 3 March 2012 - Python-checkins

cpython (3.2): Issue #14159: Fix the len() of weak containers (WeakSet, WeakKeyDictionary,
by antoine.pitrou March 1, 2012

March 1, 2012

http://hg.python.org/cpython/rev/1cd0688ff004 changeset: 75356:1cd0688ff004 branch: 3.2 parent: 75348:1c77eadba9dc user: Antoine Pitrou <solipsis(a)pitrou.net> date: Thu Mar 01 16:26:35 2012 +0100 summary: Issue #14159: Fix the len() of weak containers (WeakSet, WeakKeyDictionary, WeakValueDictionary) to return a better approximation when some objects are dead or dying. Moreover, the implementation is now O(1) rather than O(n). Thanks to Yury Selivanov for reporting. files: Lib/_weakrefset.py | 2 +- Lib/test/test_weakref.py | 60 ++++++++++++++++++++++++++++ Lib/test/test_weakset.py | 47 +++++++++++++++++++++ Lib/weakref.py | 4 +- Misc/NEWS | 5 ++ 5 files changed, 115 insertions(+), 3 deletions(-) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -63,7 +63,7 @@ yield item def __len__(self): - return sum(x() is not None for x in self.data) + return len(self.data) - len(self._pending_removals) def __contains__(self, item): try: diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -812,11 +812,71 @@ def __hash__(self): return hash(self.arg) +class RefCycle: + def __init__(self): + self.cycle = self + class MappingTestCase(TestBase): COUNT = 10 + def check_len_cycles(self, dict_type, cons): + N = 20 + items = [RefCycle() for i in range(N)] + dct = dict_type(cons(o) for o in items) + # Keep an iterator alive + it = dct.items() + try: + next(it) + except StopIteration: + pass + del items + gc.collect() + n1 = len(dct) + del it + gc.collect() + n2 = len(dct) + # one item may be kept alive inside the iterator + self.assertIn(n1, (0, 1)) + self.assertEqual(n2, 0) + + def test_weak_keyed_len_cycles(self): + self.check_len_cycles(weakref.WeakKeyDictionary, lambda k: (k, 1)) + + def test_weak_valued_len_cycles(self): + self.check_len_cycles(weakref.WeakValueDictionary, lambda k: (1, k)) + + def check_len_race(self, dict_type, cons): + # Extended sanity checks for len() in the face of cyclic collection + self.addCleanup(gc.set_threshold, *gc.get_threshold()) + for th in range(1, 100): + N = 20 + gc.collect(0) + gc.set_threshold(th, th, th) + items = [RefCycle() for i in range(N)] + dct = dict_type(cons(o) for o in items) + del items + # All items will be collected at next garbage collection pass + it = dct.items() + try: + next(it) + except StopIteration: + pass + n1 = len(dct) + del it + n2 = len(dct) + self.assertGreaterEqual(n1, 0) + self.assertLessEqual(n1, N) + self.assertGreaterEqual(n2, 0) + self.assertLessEqual(n2, n1) + + def test_weak_keyed_len_race(self): + self.check_len_race(weakref.WeakKeyDictionary, lambda k: (k, 1)) + + def test_weak_valued_len_race(self): + self.check_len_race(weakref.WeakValueDictionary, lambda k: (1, k)) + def test_weak_values(self): # # This exercises d.copy(), d.items(), d[], del d[], len(d). diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -17,6 +17,10 @@ class Foo: pass +class RefCycle: + def __init__(self): + self.cycle = self + class TestWeakSet(unittest.TestCase): @@ -359,6 +363,49 @@ s.clear() self.assertEqual(len(s), 0) + def test_len_cycles(self): + N = 20 + items = [RefCycle() for i in range(N)] + s = WeakSet(items) + del items + it = iter(s) + try: + next(it) + except StopIteration: + pass + gc.collect() + n1 = len(s) + del it + gc.collect() + n2 = len(s) + # one item may be kept alive inside the iterator + self.assertIn(n1, (0, 1)) + self.assertEqual(n2, 0) + + def test_len_race(self): + # Extended sanity checks for len() in the face of cyclic collection + self.addCleanup(gc.set_threshold, *gc.get_threshold()) + for th in range(1, 100): + N = 20 + gc.collect(0) + gc.set_threshold(th, th, th) + items = [RefCycle() for i in range(N)] + s = WeakSet(items) + del items + # All items will be collected at next garbage collection pass + it = iter(s) + try: + next(it) + except StopIteration: + pass + n1 = len(s) + del it + n2 = len(s) + self.assertGreaterEqual(n1, 0) + self.assertLessEqual(n1, N) + self.assertGreaterEqual(n2, 0) + self.assertLessEqual(n2, n1) + def test_main(verbose=None): support.run_unittest(TestWeakSet) diff --git a/Lib/weakref.py b/Lib/weakref.py --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -78,7 +78,7 @@ del self.data[key] def __len__(self): - return sum(wr() is not None for wr in self.data.values()) + return len(self.data) - len(self._pending_removals) def __contains__(self, key): try: @@ -290,7 +290,7 @@ return self.data[ref(key)] def __len__(self): - return len(self.data) + return len(self.data) - len(self._pending_removals) def __repr__(self): return "<WeakKeyDictionary at %s>" % id(self) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -127,6 +127,11 @@ Library ------- +- Issue #14159: Fix the len() of weak containers (WeakSet, WeakKeyDictionary, + WeakValueDictionary) to return a better approximation when some objects + are dead or dying. Moreover, the implementation is now O(1) rather than + O(n). + - Issue #13125: Silence spurious test_lib2to3 output when in non-verbose mode. Patch by Mikhail Novikov. -- Repository URL: http://hg.python.org/cpython

1 0

peps: PEP 416
by victor.stinner March 1, 2012

March 1, 2012

http://hg.python.org/peps/rev/1cb1bad760c6 changeset: 4102:1cb1bad760c6 user: Victor Stinner <vstinner(a)wyplay.com> date: Thu Mar 01 13:06:07 2012 +0100 summary: PEP 416 * Rephrase the first Rationale paragraph * Add 2 more use cases: cache and sandbox * Fix a typo files: pep-0416.txt | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pep-0416.txt b/pep-0416.txt --- a/pep-0416.txt +++ b/pep-0416.txt @@ -19,16 +19,20 @@ Rationale ========= -A frozendict mapping cannot be changed, but its values can be mutable (not -hashable). A frozendict is hashable and so immutable if all values are hashable -(immutable). +A frozendict is a read-only mapping: a key cannot be added nor removed, and a +key is always mapped to the same value. However, frozendict values can be +mutable (not hashable). A frozendict is hashable and so immutable if and only +if all values are hashable (immutable). Use cases of frozendict: + * frozendict can be used to implement a cache * hashable frozendict can be used as a key of a mapping or as a member of set * frozendict helps optimization because the mapping is constant * frozendict avoids the need of a lock when the frozendict is shared by multiple threads or processes, especially hashable frozendict + * frozendict helps to implement a security sandbox with read-only objects, + e.g. freeze __builtins__ mapping Constraints @@ -47,7 +51,7 @@ "Py_hash_t hash;" field * frozendict.__hash__() is implemented using hash(frozenset(self.items())) and caches the result in its private hash attribute - * Register frozendict has a collections.abc.Mapping + * Register frozendict as a collections.abc.Mapping * frozendict can be used with PyDict_GetItem(), but PyDict_SetItem() and PyDict_DelItem() raise a TypeError -- Repository URL: http://hg.python.org/peps

1 0

Daily reference leaks (87dcd948dc74): sum=0
by solipsis＠pitrou.net March 1, 2012

March 1, 2012

results for 87dcd948dc74 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogf6upG3', '-x']

1 0

cpython: update skip reason
by philip.jenvey March 1, 2012

March 1, 2012

http://hg.python.org/cpython/rev/87dcd948dc74 changeset: 75355:87dcd948dc74 user: Philip Jenvey <pjenvey(a)underboss.org> date: Wed Feb 29 16:21:25 2012 -0800 summary: update skip reason files: Lib/test/test_os.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1435,7 +1435,7 @@ @unittest.skipUnless(os.isatty(0) and (sys.platform.startswith('win') or (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET'))), - 'test requires either Windows or nl_langinfo(CODESET)') + 'test requires a tty and either Windows or nl_langinfo(CODESET)') def test_device_encoding(self): encoding = os.device_encoding(0) self.assertIsNotNone(encoding) -- Repository URL: http://hg.python.org/cpython

1 0

cpython: also skip test_device_encoding when stdin isn't a tty
by philip.jenvey March 1, 2012

March 1, 2012

http://hg.python.org/cpython/rev/dd0638da6d51 changeset: 75354:dd0638da6d51 user: Philip Jenvey <pjenvey(a)underboss.org> date: Wed Feb 29 16:16:15 2012 -0800 summary: also skip test_device_encoding when stdin isn't a tty files: Lib/test/test_os.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1433,8 +1433,8 @@ # Return None when an fd doesn't actually exist. self.assertIsNone(os.device_encoding(123456)) - @unittest.skipUnless(sys.platform.startswith('win') or - (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET')), + @unittest.skipUnless(os.isatty(0) and (sys.platform.startswith('win') or + (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET'))), 'test requires either Windows or nl_langinfo(CODESET)') def test_device_encoding(self): encoding = os.device_encoding(0) -- Repository URL: http://hg.python.org/cpython

1 0