From jython-checkins at python.org Sun Aug 2 04:04:57 2015 From: jython-checkins at python.org (stefan.richthofer) Date: Sun, 02 Aug 2015 02:04:57 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_new_gc-flags=3A_FORCE?= =?utf-8?q?=5FDELAYED=5FFINALIZATION_and?= Message-ID: <20150802020457.5519.19157@psf.io> https://hg.python.org/jython/rev/f3c97f33e089 changeset: 7714:f3c97f33e089 user: Stefan Richthofer date: Sun Aug 02 04:03:51 2015 +0200 summary: Added new gc-flags: FORCE_DELAYED_FINALIZATION and FORCE_DELAYED_WEAKREF_CALLBACKS. Also provided public methods to restore finalizers and weak references. Documentation will be added later. The new features are needed for proper gc-support in JyNI. files: Lib/test/test_gc_jy.py | 538 ++++++++- src/org/python/core/JyAttribute.java | 2 +- src/org/python/modules/_weakref/AbstractReference.java | 2 +- src/org/python/modules/_weakref/GlobalRef.java | 14 +- src/org/python/modules/gc.java | 288 +++- 5 files changed, 688 insertions(+), 156 deletions(-) diff --git a/Lib/test/test_gc_jy.py b/Lib/test/test_gc_jy.py --- a/Lib/test/test_gc_jy.py +++ b/Lib/test/test_gc_jy.py @@ -14,7 +14,7 @@ from Queue import Queue try: - from java.lang import System, Runnable, Class + from java.lang import System, Runnable, Class, Object from javatests import GCTestHelper except ImportError: #i.e. not Jython is running @@ -105,9 +105,9 @@ gc.collect() del t del l - #Note that usually two collected objects would be expected - l and t. - #But we intentionally only monitored one of them, so only one should - #be counted. + # Note that usually two collected objects would be expected - l and t. + # But we intentionally only monitored one of them, so only one should + # be counted. self.assertEqual(gc.collect(), 1) @@ -132,18 +132,18 @@ pass def test_finalization_preprocess_and_postprocess(self): - #Note that this test is done here again (already was in another class - #in this module), to see that everything works as it should also with - #a different flag-context. + # Note that this test is done here again (already was in another class + # in this module), to see that everything works as it should also with + # a different flag-context. comments = [] self0 = self class A: def __del__(self): self0.assertIn("run PreProcess", comments) comments.append("A del") - #let's simulate a time-consuming finalizer - #to ensure that post finalization processing - #is sensitive to this + # let's simulate a time-consuming finalizer + # to ensure that post finalization processing + # is sensitive to this time.sleep(0.5) comments.append("A del done") @@ -168,16 +168,16 @@ # are not in monitor-mode, i.e. gc runs async. gc.registerPreFinalizationProcess(prePr) gc.registerPostFinalizationProcess(postPr) - #Note that order matters here: - #If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used, - #gc.registerPostFinalizationProcess(postPr, 0) would lead to failure, - #because postPr asserts that a's finalizer already ran. Since - #DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess, - #to perform delayed finalization, the 0-index would prepend postPr - #before the process that actually runs the finalizers. + # Note that order matters here: + # If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used, + # gc.registerPostFinalizationProcess(postPr, 0) would lead to failure, + # because postPr asserts that a's finalizer already ran. Since + # DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess, + # to perform delayed finalization, the 0-index would prepend postPr + # before the process that actually runs the finalizers. System.gc() - #we wait a bit longer here, since PostProcess runs asynchronous - #and must wait for the finalizer of A + # we wait a bit longer here, since PostProcess runs asynchronous + # and must wait for the finalizer of A time.sleep(2) self.assertIn("run PostProcess", comments) comments = [] @@ -220,8 +220,8 @@ f = A(i) del f System.gc() - #we wait a bit longer here, since PostProcess runs asynchronous - #and must wait for the finalizer of A + # we wait a bit longer here, since PostProcess runs asynchronous + # and must wait for the finalizer of A time.sleep(4) self.assertEqual(len(comments), 8) self.assertEqual(PreProcess.preCount, 1) @@ -253,21 +253,21 @@ except Exception: pass - #Tests from GCTests_Jy_preprocess_and_postprocess are repeated here - #without monitoring. + # Tests from GCTests_Jy_preprocess_and_postprocess are repeated here + # without monitoring. def test_finalization_preprocess_and_postprocess(self): - #Note that this test is done here again (already was in another class - #in this module), to see that everything works as it should also with - #a different flag-context. + # Note that this test is done here again (already was in another class + # in this module), to see that everything works as it should also with + # a different flag-context. comments = [] self0 = self class A: def __del__(self): self0.assertIn("run PreProcess", comments) comments.append("A del") - #let's simulate a time-consuming finalizer - #to ensure that post finalization processing - #is sensitive to this + # let's simulate a time-consuming finalizer + # to ensure that post finalization processing + # is sensitive to this time.sleep(0.5) comments.append("A del done") @@ -287,20 +287,20 @@ a = None prePr = PreProcess() postPr = PostProcess() - time.sleep(1) # <- to avoid that the newly registered processes + time.sleep(2) # <- to avoid that the newly registered processes # become subject to previous run gc.registerPreFinalizationProcess(prePr) gc.registerPostFinalizationProcess(postPr) - #Note that order matters here: - #If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used, - #gc.registerPostFinalizationProcess(postPr, 0) would lead to failure, - #because postPr asserts that a's finalizer already ran. Since - #DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess, - #to perform delayed finalization, the 0-index would prepend postPr - #before the process that actually runs the finalizers. + # Note that order matters here: + # If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used, + # gc.registerPostFinalizationProcess(postPr, 0) would lead to failure, + # because postPr asserts that a's finalizer already ran. Since + # DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess, + # to perform delayed finalization, the 0-index would prepend postPr + # before the process that actually runs the finalizers. System.gc() - #we wait a bit longer here, since PostProcess runs asynchronous - #and must wait for the finalizer of A + # we wait a bit longer here, since PostProcess runs asynchronous + # and must wait for the finalizer of A time.sleep(2) self.assertIn("run PostProcess", comments) comments = [] @@ -336,15 +336,15 @@ for i in range(4): f = A(i) del f - #NastyFinalizer would cause this test occasionally to fail + # NastyFinalizer would cause this test occasionally to fail externFinalizer = GCTestHelper.NotSoNastyFinalizer() del externFinalizer for i in range(4, 8): f = A(i) del f System.gc() - #we wait a bit longer here, since PostProcess runs asynchronous - #and must wait for the finalizer of A + # we wait a bit longer here, since PostProcess runs asynchronous + # and must wait for the finalizer of A time.sleep(4) self.assertEqual(len(comments), 8) self.assertEqual(PreProcess.preCount, 1) @@ -389,13 +389,194 @@ del a del c self.assertNotEqual(gc.collect(), 0) + time.sleep(2) + # Note that CPython would collect a, b and c in one run. + # With gc.DONT_FINALIZE_RESURRECTED_OBJECTS set, Jython + # Would not collect a and b in the same run with c + # because a and b might have been resurrected by c and + # Java allows not to detect such resurrection in any + # other way than waiting for the next gc-run. + self.assertIn('del c', comments) + self.assertEqual(1, len(comments)) + comments = [] + self.assertNotEqual(gc.collect(), 0) + time.sleep(2) + self.assertIn('del a', comments) + self.assertEqual(1, len(comments)) + comments = [] + self.assertNotEqual(gc.collect(), 0) + time.sleep(2) + self.assertIn('del b', comments) + self.assertEqual(1, len(comments)) + + + at unittest.skipUnless(test_support.is_jython, + 'This class tests detailed Jython-specific behavior.') +class GCTests_Jy_Forced_Delayed_Finalization(unittest.TestCase): +# Here we basically reproduce the ordinary delayed finalization test, but ensure +# that the FORCE_DELAYED_FINALIZATION-flag does not cause regressions with this. + @classmethod + def setUpClass(cls): + #Jython-specific block: + try: + cls.savedJythonGCFlags = gc.getJythonGCFlags() + #the finalizer-related tests need this flag to pass in Jython: + gc.addJythonGCFlags(gc.DONT_FINALIZE_RESURRECTED_OBJECTS) + gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) + gc.stopMonitoring() + except Exception: + pass + + @classmethod + def tearDownClass(cls): + try: + gc.setJythonGCFlags(cls.savedJythonGCFlags) + except Exception: + pass + + # Tests from GCTests_Jy_preprocess_and_postprocess are repeated here + # without monitoring but with forced flag. + def test_forced_finalization_preprocess_and_postprocess(self): + # Note that this test is done here again (already was in another class + # in this module), to see that everything works as it should also with + # a different flag-context. + comments = [] + self0 = self + class A: + def __del__(self): + self0.assertIn("run PreProcess", comments) + comments.append("A del") + # let's simulate a time-consuming finalizer + # to ensure that post finalization processing + # is sensitive to this + time.sleep(0.5) + comments.append("A del done") + + class PreProcess(Runnable): + def run(self): + self0.assertEqual(comments, []) + comments.append("run PreProcess") + + class PostProcess(Runnable): + def run(self): + self0.assertIn("run PreProcess", comments) + self0.assertIn("A del", comments) + self0.assertIn("A del done", comments) + comments.append("run PostProcess") + + a = A() + a = None + prePr = PreProcess() + postPr = PostProcess() + time.sleep(1) # <- to avoid that the newly registered processes + # become subject to previous run + gc.registerPreFinalizationProcess(prePr) + gc.registerPostFinalizationProcess(postPr) + # Note that order matters here: + # If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used, + # gc.registerPostFinalizationProcess(postPr, 0) would lead to failure, + # because postPr asserts that a's finalizer already ran. Since + # DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess, + # to perform delayed finalization, the 0-index would prepend postPr + # before the process that actually runs the finalizers. + System.gc() + # we wait a bit longer here, since PostProcess runs asynchronous + # and must wait for the finalizer of A + time.sleep(2) + self.assertIn("run PostProcess", comments) + comments = [] + gc.unregisterPreFinalizationProcess(prePr) + gc.unregisterPostFinalizationProcess(postPr) + + def test_forced_with_extern_NonPyObjectFinalizer_that_notifies_gc(self): + comments = [] + class A: + def __init__(self, index): + self.index = index + + def __del__(self): + comments.append("A_del_"+str(self.index)) + + class PreProcess(Runnable): + preCount = 0 + def run(self): + PreProcess.preCount += 1 + + class PostProcess(Runnable): + postCount = 0 + def run(self): + PostProcess.postCount += 1 + + prePr = PreProcess() + postPr = PostProcess() + time.sleep(1) # <- to avoid that the newly registered processes + # become subject to previous run (remember: We + # are not in monitor-mode, i.e. gc runs async. + gc.registerPreFinalizationProcess(prePr) + gc.registerPostFinalizationProcess(postPr) + for i in range(4): + f = A(i) + del f + #NastyFinalizer would cause this test occasionally to fail + externFinalizer = GCTestHelper.NotSoNastyFinalizer() + del externFinalizer + for i in range(4, 8): + f = A(i) + del f + System.gc() + # we wait a bit longer here, since PostProcess runs asynchronous + # and must wait for the finalizer of A + time.sleep(4) + self.assertEqual(len(comments), 8) + self.assertEqual(PreProcess.preCount, 1) + self.assertEqual(PostProcess.postCount, 1) + comments = [] + gc.unregisterPreFinalizationProcess(prePr) + gc.unregisterPostFinalizationProcess(postPr) + + def test_forced_delayedFinalization(self): + #time.sleep(2) + resurrect = [] + comments = [] + + class Test_Finalizable(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "<"+self.name+">" + + def __del__(self): + comments.append("del "+self.name) + + class Test_Resurrection(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "<"+self.name+">" + + def __del__(self): + comments.append("del "+self.name) + if hasattr(self, "toResurrect"): + resurrect.append(self.toResurrect) + + a = Test_Finalizable("a") + a.b = Test_Finalizable("b") + c = Test_Resurrection("c") + c.a = a + c.toResurrect = Test_Finalizable("d") + + del a + del c + self.assertNotEqual(gc.collect(), 0) time.sleep(1) - #Note that CPython would collect a, b and c in one run. - #With gc.DONT_FINALIZE_RESURRECTED_OBJECTS set, Jython - #Would not collect a and b in the same run with c - #because a and b might have been resurrected by c and - #Java allows not to detect such resurrection in any - #other way than waiting for the next gc-run. + # Note that CPython would collect a, b and c in one run. + # With gc.DONT_FINALIZE_RESURRECTED_OBJECTS set, Jython + # Would not collect a and b in the same run with c + # because a and b might have been resurrected by c and + # Java allows not to detect such resurrection in any + # other way than waiting for the next gc-run. self.assertIn('del c', comments) self.assertEqual(1, len(comments)) comments = [] @@ -410,6 +591,197 @@ self.assertEqual(1, len(comments)) + at unittest.skipUnless(test_support.is_jython, + 'This class tests detailed Jython-specific behavior.') +class GCTests_Jy_Raw_Forced_Delayed_Finalization(unittest.TestCase): + + @classmethod + def setUpClass(cls): + #Jython-specific block: + try: + cls.savedJythonGCFlags = gc.getJythonGCFlags() + #the finalizer-related tests need this flag to pass in Jython: + gc.stopMonitoring() + #gc.addJythonGCFlags(gc.VERBOSE_DELAYED) + except Exception: + pass + + @classmethod + def tearDownClass(cls): + try: + gc.setJythonGCFlags(cls.savedJythonGCFlags) + except Exception: + pass + + def test_raw_forced_delayedFinalization(self): + #print "test_raw_forced_delayedFinalization" + comments = [] + + class Test_JavaAbortFinalizable(Object): + def __init__(self, name, toAbort): + self.name = name + self.toAbort = toAbort + + def __repr__(self): + return "<"+self.name+">" + + def finalize(self): + gc.notifyPreFinalization() + comments.append("del "+self.name) + gc.abortDelayedFinalization(self.toAbort) + gc.notifyPostFinalization() + + class Test_Finalizable(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "<"+self.name+">" + + def __del__(self): + comments.append("del "+self.name) + + def callback(obj): + comments.append("callback0") + + a = Test_Finalizable("a") + wa = weakref.ref(a, callback) + b = Test_JavaAbortFinalizable("b", a) + gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) + gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) + self.assertTrue(gc.delayedFinalizationEnabled()) + self.assertFalse(gc.delayedWeakrefCallbacksEnabled()) + del a + del b + System.gc() + time.sleep(2) + + self.assertIn('del b', comments) + self.assertEqual(2, len(comments)) + self.assertIn('callback0', comments) + self.assertNotIn('del a', comments) + self.assertIsNone(wa()) + gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) + + def test_raw_forced_delayedWeakrefCallback(self): + comments = [] + resurrected = [] + + class Test_JavaResurrectFinalizable(Object): + def __init__(self, name, toResurrect): + self.name = name + self.toResurrect = toResurrect + + def __repr__(self): + return "<"+self.name+">" + + # Note that this type of finalizer is usually not recommended + # as it gets lost in case of resurrection. + def finalize(self): + gc.notifyPreFinalization() + comments.append("del "+self.name) + resurrected.append(self.toResurrect) + # We manually restore weak references: + gc.restoreWeakReferences(self.toResurrect) + gc.notifyPostFinalization() + + class Test_Finalizable(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "<"+self.name+">" + + def __del__(self): + comments.append("del "+self.name) + + def callback(obj): + comments.append("callback") + + a = Test_Finalizable("a") + b = Test_JavaResurrectFinalizable("b", a) + wa = weakref.ref(a, callback) + gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) + gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) + self.assertFalse(gc.delayedFinalizationEnabled()) + self.assertTrue(gc.delayedWeakrefCallbacksEnabled()) + self.assertEqual(len(comments), 0) + aStr = str(a) + del a + del b + System.gc() + time.sleep(2) + self.assertIn("del a", comments) + self.assertIn("del b", comments) + self.assertEqual(1, len(resurrected)) + self.assertEqual(str(resurrected[0]), aStr) + self.assertIsNotNone(wa()) + self.assertEqual(resurrected[0], wa()) + self.assertNotIn("callback", comments) + self.assertEqual(2, len(comments)) + gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) + + def test_raw_forced_delayed(self): + comments = [] + + class Test_JavaAbortFinalizable(Object): + def __init__(self, name, toAbort): + self.name = name + self.toAbort = toAbort + + def __repr__(self): + return "<"+self.name+">" + + def finalize(self): + gc.notifyPreFinalization() + comments.append("del "+self.name) + gc.abortDelayedFinalization(self.toAbort) + # We manually restore weak references: + gc.restoreWeakReferences(self.toAbort) + gc.notifyPostFinalization() + + class Test_Finalizable(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "<"+self.name+">" + + def __del__(self): + comments.append("del "+self.name) + + def callback_a(obj): + comments.append("callback_a") + + def callback_b(obj): + comments.append("callback_b") + + a = Test_Finalizable("a") + wa = weakref.ref(a, callback_a) + b = Test_JavaAbortFinalizable("b", a) + wb = weakref.ref(b, callback_b) + gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) + gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) + self.assertTrue(gc.delayedFinalizationEnabled()) + self.assertTrue(gc.delayedWeakrefCallbacksEnabled()) + self.assertEqual(len(comments), 0) + del a + del b + System.gc() + time.sleep(2) + + self.assertIsNotNone(wa()) + self.assertIsNone(wb()) + self.assertIn('del b', comments) + self.assertNotIn('callback_a', comments) + self.assertIn('callback_b', comments) + self.assertNotIn('del a', comments) + self.assertEqual(2, len(comments)) + + gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) + gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) + + class GCTests_Jy_Monitoring(unittest.TestCase): @classmethod @@ -419,11 +791,11 @@ cls.savedJythonGCFlags = gc.getJythonGCFlags() gc.setMonitorGlobal(True) gc.addJythonGCFlags(gc.DONT_FINALIZE_RESURRECTED_OBJECTS) - #since gc module already exists, it would not be caught by monitorGlobal. - #so we have to monitor it manually: + # since gc module already exists, it would not be caught by monitorGlobal. + # so we have to monitor it manually: gc.monitorObject(gc) - #the finalizer-related tests need this flag to pass in Jython: - #gc.addJythonGCFlags(gc.DONT_FINALIZE_CYCLIC_GARBAGE) + # the finalizer-related tests need this flag to pass in Jython: + # gc.addJythonGCFlags(gc.DONT_FINALIZE_CYCLIC_GARBAGE) except Exception: pass @@ -477,9 +849,9 @@ self.assertEqual(gc.collect(), 0) #c is not cyclic and a, b are resurrected, #so nothing to count here #self.asserEqual(len(gc.garbage), 0) - #if we called gc.set_debug(gc.DEBUG_SAVEALL) above, it would - #be okay for gc.garbage to be empty, because a and b - #are not finalized and c is not cyclic. + # if we called gc.set_debug(gc.DEBUG_SAVEALL) above, it would + # be okay for gc.garbage to be empty, because a and b + # are not finalized and c is not cyclic. self.assertEqual(comments, ['del c']) self.assertEqual(str(resurrect), "[]") self.assertTrue(gc.isMonitored(resurrect[0])) @@ -723,7 +1095,7 @@ except Exception: pass - def test_weakref_after_resurrection_threadsafe(self): + def test_TraverseByReflection(self): gc.collect() prt = GCTestHelper.reflectionTraverseTestField() @@ -787,3 +1159,57 @@ if __name__ == "__main__": unittest.main() +# comments = [] +# resurrected = [] +# +# class Test_JavaResurrectFinalizable(Object): +# def __init__(self, name, toResurrect): +# self.name = name +# self.toResurrect = toResurrect +# +# def __repr__(self): +# return "<"+self.name+">" +# +# #def __del__(self): +# def finalize(self): +# gc.notifyPreFinalization() +# comments.append("del "+self.name) +# #gc.abortDelayedFinalization(self.toAbort) +# resurrected.append(self.toResurrect) +# print "finalize "+self.name +# # We manually restore weak references: +# gc.restoreWeakReferences(self.toResurrect) +# gc.notifyPostFinalization() +# +# class Test_Finalizable(object): +# def __init__(self, name): +# self.name = name +# +# def __repr__(self): +# return "<"+self.name+">" +# +# def __del__(self): +# comments.append("del "+self.name) +# +# def callback(obj): +# comments.append("callback")#+str(obj)) +# print "callback: "+str(obj) +# +# a = Test_Finalizable("a") +# b = Test_JavaResurrectFinalizable("b", a) +# wa = weakref.ref(a, callback) +# print ("wref: ")+str(wa()) +# gc.addJythonGCFlags(gc.VERBOSE_DELAYED) +# #gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) +# #gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) +# print "delayed finalization? "+str(gc.delayedFinalizationEnabled()) +# print "delayed callbacks? "+str(gc.delayedWeakrefCallbacksEnabled()) +# print comments +# del a +# del b +# System.gc() +# time.sleep(1) +# print comments +# print resurrected +# print ("wref: ")+str(wa()) + diff --git a/src/org/python/core/JyAttribute.java b/src/org/python/core/JyAttribute.java --- a/src/org/python/core/JyAttribute.java +++ b/src/org/python/core/JyAttribute.java @@ -81,7 +81,7 @@ * finalizable objects that might have been resurrected * during a delayed finalization process. */ - public static final byte GC_DELAYED_FINALIZE_CRITIC_MARK_ATTR = 5; + public static final byte GC_DELAYED_FINALIZE_CRITICAL_MARK_ATTR = 5; public static final byte FINALIZE_TRIGGER_ATTR = Byte.MAX_VALUE; private static byte nonBuiltinAttrTypeOffset = Byte.MIN_VALUE+1; diff --git a/src/org/python/modules/_weakref/AbstractReference.java b/src/org/python/modules/_weakref/AbstractReference.java --- a/src/org/python/modules/_weakref/AbstractReference.java +++ b/src/org/python/modules/_weakref/AbstractReference.java @@ -79,7 +79,7 @@ protected PyObject get() { PyObject result = gref.get(); - if (result == null && (gc.getJythonGCFlags() & gc.PRESERVE_WEAKREFS_ON_RESURRECTION) != 0) { + if (result == null && gc.delayedWeakrefCallbacksEnabled()) { if (gref.cleared) { return null; } diff --git a/src/org/python/modules/_weakref/GlobalRef.java b/src/org/python/modules/_weakref/GlobalRef.java --- a/src/org/python/modules/_weakref/GlobalRef.java +++ b/src/org/python/modules/_weakref/GlobalRef.java @@ -208,7 +208,8 @@ * @throws java.lang.IllegalArgumentException if {@code formerReferent} is not * the actual former referent. */ - public void restore(PyObject formerReferent) { + public synchronized void restore(PyObject formerReferent) { + /* This method is synchronized to avoid concurrent invocation of call(). */ if (JyAttribute.getAttr(formerReferent, JyAttribute.WEAK_REF_ATTR) != this) { throw new IllegalArgumentException( "Argument is not former referent of this GlobalRef."); @@ -238,6 +239,15 @@ } } } + /* We must clear the old global ref to avoid processing of the + * callback in spite of restore (might happen because of bad timing). + * (The remove from delayed callback list might happen before + * the insert.) However we can only set cleared = true after + * all gref-variables were updated, otherwise some refs might + * break. To avoid callback-rocessing in the unsafe state + * between these actions, this method is synchronized (as is call()). + */ + cleared = true; } private static void createReaperThreadIfAbsent() { @@ -351,7 +361,7 @@ public void collect() throws InterruptedException { GlobalRef gr = (GlobalRef) referenceQueue.remove(); - if ((gc.getJythonGCFlags() & gc.PRESERVE_WEAKREFS_ON_RESURRECTION) == 0) { + if (!gc.delayedWeakrefCallbacksEnabled()) { gr.call(); } else { delayedCallback(gr); diff --git a/src/org/python/modules/gc.java b/src/org/python/modules/gc.java --- a/src/org/python/modules/gc.java +++ b/src/org/python/modules/gc.java @@ -247,6 +247,9 @@ */ public static final short DONT_FINALIZE_RESURRECTED_OBJECTS = (1<<3); + public static final short FORCE_DELAYED_FINALIZATION = (1<<4); + public static final short FORCE_DELAYED_WEAKREF_CALLBACKS = (1<<5); + /** *

* Reflection-based traversion is an inefficient fallback-method to @@ -283,7 +286,7 @@ * @see #SUPPRESS_TRAVERSE_BY_REFLECTION_WARNING * @see #INSTANCE_TRAVERSE_BY_REFLECTION_WARNING */ - public static final short DONT_TRAVERSE_BY_REFLECTION = (1<<4); + public static final short DONT_TRAVERSE_BY_REFLECTION = (1<<6); /** *

@@ -310,7 +313,7 @@ * @see #removeJythonGCFlags(short) * @see #INSTANCE_TRAVERSE_BY_REFLECTION_WARNING */ - public static final short SUPPRESS_TRAVERSE_BY_REFLECTION_WARNING = (1<<5); + public static final short SUPPRESS_TRAVERSE_BY_REFLECTION_WARNING = (1<<7); /** * Makes gc emit reflection-based traversion warning for every traversed @@ -325,7 +328,7 @@ * @see #removeJythonGCFlags(short) * @see #SUPPRESS_TRAVERSE_BY_REFLECTION_WARNING */ - public static final short INSTANCE_TRAVERSE_BY_REFLECTION_WARNING = (1<<6); + public static final short INSTANCE_TRAVERSE_BY_REFLECTION_WARNING = (1<<8); /** * In Jython one usually uses {@code Py.writeDebug} for debugging output. @@ -341,7 +344,7 @@ * @see #addJythonGCFlags(short) * @see #removeJythonGCFlags(short) */ - public static final short USE_PY_WRITE_DEBUG = (1<<7); + public static final short USE_PY_WRITE_DEBUG = (1<<9); /** * Enables collection-related verbose-output. @@ -351,7 +354,7 @@ * @see #addJythonGCFlags(short) * @see #removeJythonGCFlags(short) */ - public static final short VERBOSE_COLLECT = (1<<8); + public static final short VERBOSE_COLLECT = (1<<10); /** * Enables weakref-related verbose-output. @@ -361,7 +364,7 @@ * @see #addJythonGCFlags(short) * @see #removeJythonGCFlags(short) */ - public static final short VERBOSE_WEAKREF = (1<<9); + public static final short VERBOSE_WEAKREF = (1<<11); /** * Enables delayed finalization related verbose-output. @@ -371,7 +374,7 @@ * @see #addJythonGCFlags(short) * @see #removeJythonGCFlags(short) */ - public static final short VERBOSE_DELAYED = (1<<10); + public static final short VERBOSE_DELAYED = (1<<12); /** * Enables finalization-related verbose-output. @@ -381,7 +384,7 @@ * @see #addJythonGCFlags(short) * @see #removeJythonGCFlags(short) */ - public static final short VERBOSE_FINALIZE = (1<<11); + public static final short VERBOSE_FINALIZE = (1<<13); /** * Bit-combination of the flags {@link #VERBOSE_COLLECT}, @@ -495,11 +498,11 @@ private static boolean lockPostFinalization = false; /* Resurrection-safe finalizer- and weakref-related declarations: */ - private static IdentityHashMap delayedFinalizables, resurrectionCritics; + private static IdentityHashMap delayedFinalizables, resurrectionCriticals; private static int abortedCyclicFinalizers = 0; /* Some modes to control aspects of delayed finalization: */ private static final byte DO_NOTHING_SPECIAL = 0; - private static final byte MARK_REACHABLE_CRITICS = 1; + private static final byte MARK_REACHABLE_CRITICALS = 1; private static final byte NOTIFY_FOR_RERUN = 2; private static byte delayedFinalizationMode = DO_NOTHING_SPECIAL; private static boolean notifyRerun = false; @@ -822,11 +825,95 @@ //--------------delayed finalization section----------------------------------- + /** + * In addition to what + * {@link org.python.core.finalization.FinalizeTrigger#ensureFinalizer(PyObject)} + * does, this method also restores the finalizer's + * {@link org.python.core.finalization.FinalizeTrigger}'s flags by taking the + * values from the former finalizer. On the other hand - in contrast to + * {@link org.python.core.finalization.FinalizeTrigger#ensureFinalizer(PyObject)} - + * this method would not create a {@link org.python.core.finalization.FinalizeTrigger} + * for an object that did not have one before (i.e. the method checks for an old + * (dead) trigger before it creates a new one.

+ * If a new finalizer is needed due to an + * ordinary resurrection (i.e. the object's finalizer was called), + * {@link org.python.core.finalization.FinalizeTrigger#ensureFinalizer(PyObject)} + * is the right choice. If a finalization was vetoed in context of delayed + * finalization (i.e. a resurrection that pretends not to be one and didn't run + * the finalizer), this method is the better choice as it helps to make the new + * {@link org.python.core.finalization.FinalizeTrigger} look exactly like the + * old one regarding flags etc. + * E.g. this method is called by {@link #abortDelayedFinalization(PyObject)}. + * + * @see #abortDelayedFinalization(PyObject) + */ + public static void restoreFinalizer(PyObject obj) { + FinalizeTrigger ft = + (FinalizeTrigger) JyAttribute.getAttr(obj, JyAttribute.FINALIZE_TRIGGER_ATTR); + boolean notify = false; + if (ft != null) { + FinalizeTrigger.ensureFinalizer(obj); + /* ensure that the old finalize won't run in any case */ + ft.clear(); + ((FinalizeTrigger) JyAttribute.getAttr(obj, + JyAttribute.FINALIZE_TRIGGER_ATTR)).flags = ft.flags; + notify = (ft.flags & FinalizeTrigger.NOTIFY_GC_FLAG) != 0; + } + if ((gcFlags & VERBOSE_DELAYED) != 0 || (gcFlags & VERBOSE_FINALIZE) != 0) { + writeDebug("gc", "restore finalizer of "+obj); + } + CycleMarkAttr cm = (CycleMarkAttr) + JyAttribute.getAttr(obj, JyAttribute.GC_CYCLE_MARK_ATTR); + if (cm != null && cm.monitored) { + monitorObject(obj, true); + } + if (notify) { + + boolean cyclic; + if (cm != null && cm.isUncollectable()) { + cyclic = true; + } else { + markCyclicObjects(obj, true); + cm = (CycleMarkAttr) JyAttribute.getAttr(obj, JyAttribute.GC_CYCLE_MARK_ATTR); + cyclic = cm != null && cm.isUncollectable(); + } + + if ((gcFlags & VERBOSE_DELAYED) != 0 || (gcFlags & VERBOSE_FINALIZE) != 0) { + writeDebug("gc", "notify finalizer abort; cyclic? "+cyclic); + } + notifyAbortFinalize(obj, cyclic); + } + } + + /** + * Restores weak references pointing to {@code rst}. Note that + * this does not prevent callbacks, unless it is called during + * finalization phase (e.g. by a finalizer) and + * {@link #delayedWeakrefCallbacksEnabled()} returns {@code true}. + * In a manual fashion, one can enforce this by using the gc-flag + * {@link #FORCE_DELAYED_WEAKREF_CALLBACKS}. Alternatively, one can + * use the automatic way via the gc-flag + * {@link #PRESERVE_WEAKREFS_ON_RESURRECTION}, but then one would + * not need to call this method anyway. The manual way has better + * performance, but also brings more responsibilies. + * + * @see #delayedWeakrefCallbacksEnabled() + * @see #FORCE_DELAYED_WEAKREF_CALLBACKS + * @see #PRESERVE_WEAKREFS_ON_RESURRECTION + */ + public static void restoreWeakReferences(PyObject rst) { + GlobalRef toRestore = (GlobalRef) + JyAttribute.getAttr(rst, JyAttribute.WEAK_REF_ATTR); + if (toRestore != null) { + toRestore.restore(rst); + } + } + private static class DelayedFinalizationProcess implements Runnable { static DelayedFinalizationProcess defaultInstance = new DelayedFinalizationProcess(); - private void performFinalization(PyObject del) { + private static void performFinalization(PyObject del) { if ((gcFlags & VERBOSE_DELAYED) != 0) { writeDebug("gc", "delayed finalize of "+del); } @@ -839,110 +926,98 @@ } } - private void restoreFinalizer(PyObject obj, boolean cyclic) { - FinalizeTrigger ft = - (FinalizeTrigger) JyAttribute.getAttr(obj, JyAttribute.FINALIZE_TRIGGER_ATTR); - FinalizeTrigger.ensureFinalizer(obj); - boolean notify = false; - if (ft != null) { - ((FinalizeTrigger) - JyAttribute.getAttr(obj, JyAttribute.FINALIZE_TRIGGER_ATTR)).flags - = ft.flags; - notify = (ft.flags & FinalizeTrigger.NOTIFY_GC_FLAG) != 0; - } - if ((gcFlags & VERBOSE_DELAYED) != 0 || (gcFlags & VERBOSE_FINALIZE) != 0) { - writeDebug("gc", "restore finalizer of "+obj+"; cyclic? "+cyclic); - } - CycleMarkAttr cm = (CycleMarkAttr) - JyAttribute.getAttr(obj, JyAttribute.GC_CYCLE_MARK_ATTR); - if (cm != null && cm.monitored) { - monitorObject(obj, true); - } - if (notify) { - if ((gcFlags & VERBOSE_DELAYED) != 0 || (gcFlags & VERBOSE_FINALIZE) != 0) { - writeDebug("gc", "notify finalizer abort."); - } - notifyAbortFinalize(obj, cyclic); - } - } - public void run() { if ((gcFlags & VERBOSE_DELAYED) != 0) { writeDebug("gc", "run delayed finalization. Index: "+ gcMonitoredRunCount); } - Set critics = resurrectionCritics.keySet(); - Set cyclicCritics = removeNonCyclic(critics); - cyclicCritics.retainAll(critics); - critics.removeAll(cyclicCritics); - Set criticReachablePool = findReachables(critics); + Set criticals = resurrectionCriticals.keySet(); + if (delayedFinalizationMode == DO_NOTHING_SPECIAL && + (gcFlags & (PRESERVE_WEAKREFS_ON_RESURRECTION | + DONT_FINALIZE_RESURRECTED_OBJECTS)) == 0) { + /* In this case we can do a cheap variant... */ + if ((gcFlags & FORCE_DELAYED_WEAKREF_CALLBACKS) != 0) { + if ((gcFlags & VERBOSE_DELAYED) != 0) { + writeDebug("gc", "process delayed callbacks (force-branch)"); + } + GlobalRef.processDelayedCallbacks(); + } + if ((gcFlags & FORCE_DELAYED_FINALIZATION) != 0) { + if ((gcFlags & VERBOSE_DELAYED) != 0) { + writeDebug("gc", "process delayed finalizers (force-branch)"); + } + for (PyObject del: delayedFinalizables.keySet()) { + performFinalization(del); + } + for (PyObject cr: criticals) { + performFinalization(cr); + } + delayedFinalizables.clear(); + resurrectionCriticals.clear(); + } + if ((gcFlags & VERBOSE_DELAYED) != 0) { + writeDebug("gc", "forced delayed finalization run done"); + } + return; + } + + Set cyclicCriticals = removeNonCyclic(criticals); + cyclicCriticals.retainAll(criticals); + criticals.removeAll(cyclicCriticals); + Set criticalReachablePool = findReachables(criticals); /* to avoid concurrent modification: */ - ArrayList criticReachables = new ArrayList<>(); + ArrayList criticalReachables = new ArrayList<>(); FinalizeTrigger fn; - if (delayedFinalizationMode == MARK_REACHABLE_CRITICS) { - for (PyObject obj: criticReachablePool) { + if (delayedFinalizationMode == MARK_REACHABLE_CRITICALS) { + for (PyObject obj: criticalReachablePool) { fn = (FinalizeTrigger) JyAttribute.getAttr(obj, JyAttribute.FINALIZE_TRIGGER_ATTR); if (fn != null && fn.isActive() && fn.isFinalized()) { - criticReachables.add(obj); + criticalReachables.add(obj); JyAttribute.setAttr(obj, - JyAttribute.GC_DELAYED_FINALIZE_CRITIC_MARK_ATTR, + JyAttribute.GC_DELAYED_FINALIZE_CRITICAL_MARK_ATTR, Integer.valueOf(gcMonitoredRunCount)); } } } else { - for (PyObject obj: criticReachablePool) { + for (PyObject obj: criticalReachablePool) { fn = (FinalizeTrigger) JyAttribute.getAttr(obj, JyAttribute.FINALIZE_TRIGGER_ATTR); if (fn != null && fn.isActive() && fn.isFinalized()) { - criticReachables.add(obj); + criticalReachables.add(obj); } } } - critics.removeAll(criticReachables); + criticals.removeAll(criticalReachables); if ((gcFlags & PRESERVE_WEAKREFS_ON_RESURRECTION) != 0) { if ((gcFlags & VERBOSE_DELAYED) != 0) { writeDebug("gc", "restore potentially resurrected weak references..."); } - GlobalRef toRestore; - for (PyObject rst: criticReachablePool) { - toRestore = (GlobalRef) - JyAttribute.getAttr(rst, JyAttribute.WEAK_REF_ATTR); - if (toRestore != null) { - toRestore.restore(rst); - } + for (PyObject rst: criticalReachablePool) { + restoreWeakReferences(rst); } GlobalRef.processDelayedCallbacks(); } - criticReachablePool.clear(); + criticalReachablePool.clear(); if ((gcFlags & DONT_FINALIZE_RESURRECTED_OBJECTS) != 0) { /* restore all finalizers that might belong to resurrected objects: */ if ((gcFlags & VERBOSE_DELAYED) != 0) { - writeDebug("gc", "restore "+criticReachables.size()+ + writeDebug("gc", "restore "+criticalReachables.size()+ " potentially resurrected finalizers..."); } - for (PyObject obj: criticReachables) { - CycleMarkAttr cm = (CycleMarkAttr) - JyAttribute.getAttr(obj, JyAttribute.GC_CYCLE_MARK_ATTR); - if (cm != null && cm.isUncollectable()) { - restoreFinalizer(obj, true); - } else { - gc.markCyclicObjects(obj, true); - cm = (CycleMarkAttr) - JyAttribute.getAttr(obj, JyAttribute.GC_CYCLE_MARK_ATTR); - restoreFinalizer(obj, cm != null && cm.isUncollectable()); - } + for (PyObject obj: criticalReachables) { + restoreFinalizer(obj); } } else { if ((gcFlags & VERBOSE_DELAYED) != 0) { - writeDebug("gc", "delayed finalization of "+criticReachables.size()+ + writeDebug("gc", "delayed finalization of "+criticalReachables.size()+ " potentially resurrected finalizers..."); } - for (PyObject del: criticReachables) { + for (PyObject del: criticalReachables) { performFinalization(del); } } - cyclicCritics.removeAll(criticReachables); + cyclicCriticals.removeAll(criticalReachables); if ((gcFlags & VERBOSE_DELAYED) != 0 && !delayedFinalizables.isEmpty()) { writeDebug("gc", "process "+delayedFinalizables.size()+ " delayed finalizers..."); @@ -950,19 +1025,19 @@ for (PyObject del: delayedFinalizables.keySet()) { performFinalization(del); } - if ((gcFlags & VERBOSE_DELAYED) != 0 && !cyclicCritics.isEmpty()) { - writeDebug("gc", "process "+cyclicCritics.size()+" cyclic delayed finalizers..."); + if ((gcFlags & VERBOSE_DELAYED) != 0 && !cyclicCriticals.isEmpty()) { + writeDebug("gc", "process "+cyclicCriticals.size()+" cyclic delayed finalizers..."); } - for (PyObject del: cyclicCritics) { + for (PyObject del: cyclicCriticals) { performFinalization(del); } - if ((gcFlags & VERBOSE_DELAYED) != 0 && !critics.isEmpty()) { - writeDebug("gc", "calling "+critics.size()+ - " critic finalizers not reachable by other critic finalizers..."); + if ((gcFlags & VERBOSE_DELAYED) != 0 && !criticals.isEmpty()) { + writeDebug("gc", "calling "+criticals.size()+ + " critical finalizers not reachable by other critical finalizers..."); } - if (delayedFinalizationMode == MARK_REACHABLE_CRITICS && - !critics.isEmpty() && !criticReachables.isEmpty()) { - /* This means some critic-reachables might be not critic-reachable any more. + if (delayedFinalizationMode == MARK_REACHABLE_CRITICALS && + !criticals.isEmpty() && !criticalReachables.isEmpty()) { + /* This means some critical-reachables might be not critical-reachable any more. * In a synchronized gc collection approach System.gc should run again while * something like this is found. (Yes, not exactly a cheap task, but since this * is for debugging, correctness counts.) @@ -970,10 +1045,10 @@ notifyRerun = true; } if (delayedFinalizationMode == NOTIFY_FOR_RERUN && !notifyRerun) { - for (PyObject del: critics) { + for (PyObject del: criticals) { if (!notifyRerun) { Object m = JyAttribute.getAttr(del, - JyAttribute.GC_DELAYED_FINALIZE_CRITIC_MARK_ATTR); + JyAttribute.GC_DELAYED_FINALIZE_CRITICAL_MARK_ATTR); if (m != null && ((Integer) m).intValue() == gcMonitoredRunCount) { notifyRerun = true; } @@ -981,31 +1056,43 @@ performFinalization(del); } } else { - for (PyObject del: critics) { + for (PyObject del: criticals) { performFinalization(del); } } delayedFinalizables.clear(); - resurrectionCritics.clear(); + resurrectionCriticals.clear(); if ((gcFlags & VERBOSE_DELAYED) != 0) { writeDebug("gc", "delayed finalization run done"); } } } + public static boolean delayedWeakrefCallbacksEnabled() { + return (gcFlags & (PRESERVE_WEAKREFS_ON_RESURRECTION | + FORCE_DELAYED_WEAKREF_CALLBACKS)) != 0; + } + public static boolean delayedFinalizationEnabled() { return (gcFlags & (PRESERVE_WEAKREFS_ON_RESURRECTION | - DONT_FINALIZE_RESURRECTED_OBJECTS)) != 0; + DONT_FINALIZE_RESURRECTED_OBJECTS | + FORCE_DELAYED_FINALIZATION)) != 0; } private static void updateDelayedFinalizationState() { - if (delayedFinalizationEnabled()) { + /* + * There might be the case where delayed weakref callbacks are enabled, + * but not delayed finalization. We still register a DelayedFinalizationProcess + * then. That process detects the situation by checking the flags and only + * performs GlobalRef.processDelayedCallbacks() then. + */ + if (delayedFinalizationEnabled() || delayedWeakrefCallbacksEnabled()) { resumeDelayedFinalization(); } else if (indexOfPostFinalizationProcess( DelayedFinalizationProcess.defaultInstance) != -1) { suspendDelayedFinalization(); } - if ((gcFlags & PRESERVE_WEAKREFS_ON_RESURRECTION) == 0) { + if (delayedWeakrefCallbacksEnabled()) { if (GlobalRef.hasDelayedCallbacks()) { Thread dlcProcess = new Thread() { public void run() { @@ -1021,8 +1108,8 @@ if (delayedFinalizables == null) { delayedFinalizables = new IdentityHashMap<>(); } - if (resurrectionCritics == null) { - resurrectionCritics = new IdentityHashMap<>(); + if (resurrectionCriticals == null) { + resurrectionCriticals = new IdentityHashMap<>(); } /* add post-finalization process (and cancel pending suspension process if any) */ try { @@ -1046,18 +1133,27 @@ DelayedFinalizationProcess.defaultInstance); } - private static boolean isResurrectionCritic(PyObject ob) { + private static boolean isResurrectionCritical(PyObject ob) { return (isTraversable(ob)) && FinalizeTrigger.hasActiveTrigger(ob); } public static void registerForDelayedFinalization(PyObject ob) { - if (isResurrectionCritic(ob)) { - resurrectionCritics.put(ob, ob); + if (isResurrectionCritical(ob)) { + resurrectionCriticals.put(ob, ob); } else { delayedFinalizables.put(ob, ob); } } + + public static void abortDelayedFinalization(PyObject ob) { + resurrectionCriticals.remove(ob); + delayedFinalizables.remove(ob); + if ((gcFlags & VERBOSE_DELAYED) != 0 || (gcFlags & VERBOSE_FINALIZE) != 0) { + writeDebug("gc", "abort delayed finalization of "+ob); + } + restoreFinalizer(ob); + } //--------------end of delayed finalization section---------------------------- @@ -1814,7 +1910,7 @@ lst.clear(); } ++gcMonitoredRunCount; - delayedFinalizationMode = MARK_REACHABLE_CRITICS; + delayedFinalizationMode = MARK_REACHABLE_CRITICALS; notifyRerun = false; int[] stat = {0, 0}; -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Aug 6 18:18:51 2015 From: jython-checkins at python.org (stefan.richthofer) Date: Thu, 06 Aug 2015 16:18:51 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_necessary_hooks_for_w?= =?utf-8?q?eak_reference-support_in_JyNI=2E?= Message-ID: <20150806161851.1391.41380@psf.io> https://hg.python.org/jython/rev/d3ccce2e3db5 changeset: 7715:d3ccce2e3db5 user: Stefan Richthofer date: Thu Aug 06 17:17:16 2015 +0200 summary: Added necessary hooks for weak reference-support in JyNI. files: src/org/python/modules/_weakref/AbstractReference.java | 17 +- src/org/python/modules/_weakref/CallableProxyType.java | 2 +- src/org/python/modules/_weakref/GlobalRef.java | 119 +++++++-- src/org/python/modules/_weakref/ProxyType.java | 4 +- src/org/python/modules/_weakref/ReferenceBackend.java | 15 + src/org/python/modules/_weakref/ReferenceBackendFactory.java | 13 + src/org/python/modules/_weakref/ReferenceType.java | 6 +- src/org/python/modules/_weakref/ReferenceTypeDerived.java | 2 +- src/org/python/modules/_weakref/WeakrefModule.java | 2 +- src/org/python/modules/gc.java | 3 +- src/templates/weakref.derived | 2 +- 11 files changed, 144 insertions(+), 41 deletions(-) diff --git a/src/org/python/modules/_weakref/AbstractReference.java b/src/org/python/modules/_weakref/AbstractReference.java --- a/src/org/python/modules/_weakref/AbstractReference.java +++ b/src/org/python/modules/_weakref/AbstractReference.java @@ -16,9 +16,9 @@ PyObject callback; - protected GlobalRef gref; + protected ReferenceBackend gref; - public AbstractReference(PyType subType, GlobalRef gref, PyObject callback) { + public AbstractReference(PyType subType, ReferenceBackend gref, PyObject callback) { super(subType); this.gref = gref; this.callback = callback; @@ -49,6 +49,10 @@ return ob_other == this; } + public boolean hasCallback() { + return callback != null; + } + public int hashCode() { return gref.pythonHashCode(); } @@ -80,15 +84,16 @@ protected PyObject get() { PyObject result = gref.get(); if (result == null && gc.delayedWeakrefCallbacksEnabled()) { - if (gref.cleared) { + if (gref.isCleared()) { return null; } if ((gc.getJythonGCFlags() & gc.VERBOSE_WEAKREF) != 0) { gc.writeDebug("gc", "pending in get of abstract ref "+this+": "+ Thread.currentThread().getId()); } - JyAttribute.setAttr(this, JyAttribute.WEAKREF_PENDING_GET_ATTR, Thread.currentThread()); - while (!gref.cleared && result == null) { + JyAttribute.setAttr(this, JyAttribute.WEAKREF_PENDING_GET_ATTR, + Thread.currentThread()); + while (!gref.isCleared() && result == null) { try { Thread.sleep(2000); } catch (InterruptedException ie) {} @@ -98,7 +103,7 @@ if ((gc.getJythonGCFlags() & gc.VERBOSE_WEAKREF) != 0) { gc.writeDebug("gc", "pending of "+this+" resolved: "+ Thread.currentThread().getId()); - if (gref.cleared) { + if (gref.isCleared()) { gc.writeDebug("gc", "reference was cleared."); } else if (result != null){ gc.writeDebug("gc", "reference was restored."); diff --git a/src/org/python/modules/_weakref/CallableProxyType.java b/src/org/python/modules/_weakref/CallableProxyType.java --- a/src/org/python/modules/_weakref/CallableProxyType.java +++ b/src/org/python/modules/_weakref/CallableProxyType.java @@ -14,7 +14,7 @@ public static final PyType TYPE = PyType.fromClass(CallableProxyType.class); - public CallableProxyType(GlobalRef ref, PyObject callback) { + public CallableProxyType(ReferenceBackend ref, PyObject callback) { super(TYPE, ref, callback); } diff --git a/src/org/python/modules/_weakref/GlobalRef.java b/src/org/python/modules/_weakref/GlobalRef.java --- a/src/org/python/modules/_weakref/GlobalRef.java +++ b/src/org/python/modules/_weakref/GlobalRef.java @@ -17,7 +17,13 @@ import org.python.util.Generic; import org.python.modules.gc; -public class GlobalRef extends WeakReference { +public class GlobalRef extends WeakReference implements ReferenceBackend { + + /** + * This is a hook for JyNI to insert a native-objects-aware implementation + * of ReferenceBackend. + */ + public static ReferenceBackendFactory factory = null; /** * This reference's hashCode: The {@code System.identityHashCode} of the referent. @@ -54,7 +60,7 @@ private static Thread reaperThread; private static ReentrantReadWriteLock reaperLock = new ReentrantReadWriteLock(); - private static ConcurrentMap objects = Generic.concurrentMap(); + private static ConcurrentMap objects = Generic.concurrentMap(); private static List delayedCallbacks; public GlobalRef(PyObject object) { @@ -76,7 +82,7 @@ * Search for a reusable reference. To be reused, it must be of the * same class and it must not have a callback. */ - synchronized AbstractReference find(Class cls) { + public synchronized AbstractReference find(Class cls) { for (int i = references.size() - 1; i >= 0; i--) { AbstractReference r = getReferenceAt(i); if (r == null) { @@ -107,6 +113,12 @@ r.call(); } } + ReferenceBackend ref2 = objects.get(this); + if (ref2.isCleared()) { + objects.remove(this); + } else if (factory != null && ref2 != this) { + factory.notifyClear(ref2, this); + } } } @@ -135,6 +147,7 @@ * determined whether a resurrection restored the reference. * * @see gc#PRESERVE_WEAKREFS_ON_RESURRECTION + * @see gc#FORCE_DELAYED_WEAKREF_CALLBACKS */ private static void delayedCallback(GlobalRef cl) { if (delayedCallbacks == null) { @@ -149,6 +162,10 @@ return delayedCallbacks != null && !delayedCallbacks.isEmpty(); } + public boolean isCleared() { + return cleared; + } + synchronized public int count() { for (int i = references.size() - 1; i >= 0; i--) { AbstractReference r = getReferenceAt(i); @@ -173,26 +190,70 @@ } /** - * Create a new tracked {@code GlobalRef}. + * Returns null if nothing is changed. If a factory exists + * and produces a result different from {@code this}, this + * result is returned. Also, this result is then installed + * in all weak-references, in the referent's JyAttribute and + * in the objects-map to act as a proxy for this GlobalRef, + * which will still serve as a backend for the proxy. This + * method is most likely used exclusively by JyNI. + */ + synchronized protected ReferenceBackend retryFactory() { + if (factory == null) { + return null; + } + ReferenceBackend result = factory.makeBackend(this, null); + if (result != this) { + objects.put(this, result); + for (int i = references.size() - 1; i >= 0; i--) { + AbstractReference r = getReferenceAt(i); + if (r == null) { + references.remove(i); + } else { + r.gref = result; + } + } + PyObject referent = result.get(); + JyAttribute.setAttr(referent, JyAttribute.WEAK_REF_ATTR, result); + return result; + } + return null; + } + + /** + * Create a new tracked {@code ReferenceBackend}. + * If no {@code ReferenceBackendFactory} is registered, it actually + * returns a {@code GlobalRef}. * * @param object a {@link org.python.core.PyObject} to reference - * @return a new tracked {@code GlobalRef} + * @return a new tracked {@code ReferenceBackend} */ - public static GlobalRef newInstance(PyObject object) { + public static ReferenceBackend newInstance(PyObject object) { createReaperThreadIfAbsent(); - GlobalRef newRef = new GlobalRef(object); - GlobalRef ref = objects.putIfAbsent(newRef, newRef); - if (ref == null) { - ref = newRef; - JyAttribute.setAttr(object, JyAttribute.WEAK_REF_ATTR, ref); - } else { - // We clear the not-needed Global ref so that it won't - // pop up in ref-reaper thread's activity. - newRef.clear(); - newRef.cleared = true; + /* + * Note: Before factory was introduced, the following used to be + * objects.putIfAbsent(newRef, newRef), which is an atomic + * operation and was used to prevent multiple threads from + * creating multiple GlobalRefs for the same referent. With + * factory we cannot use objects.putIfAbsent any more, so + * we use a synchronized block instead. (Maybe objects could + * now be replaced by an ordinary map rather than concurrent.) + */ + synchronized (objects) { + ReferenceBackend ref = objects.get(newRef); + if (ref == null) { + ref = factory == null ? newRef : factory.makeBackend(newRef, object); + objects.put(newRef, ref); + JyAttribute.setAttr(object, JyAttribute.WEAK_REF_ATTR, ref); + } else { + // We clear the not-needed Global ref so that it won't + // pop up in ref-reaper thread's activity. + newRef.clear(); + newRef.cleared = true; + } + return ref; } - return ref; } /** @@ -210,7 +271,10 @@ */ public synchronized void restore(PyObject formerReferent) { /* This method is synchronized to avoid concurrent invocation of call(). */ - if (JyAttribute.getAttr(formerReferent, JyAttribute.WEAK_REF_ATTR) != this) { + ReferenceBackend formerBackend = (ReferenceBackend) + JyAttribute.getAttr(formerReferent, JyAttribute.WEAK_REF_ATTR); + ReferenceBackend proxy = objects.get(this); + if (formerBackend != this && formerBackend != proxy) { throw new IllegalArgumentException( "Argument is not former referent of this GlobalRef."); } @@ -222,16 +286,23 @@ clear(); createReaperThreadIfAbsent(); GlobalRef restore = new GlobalRef(formerReferent); + if (proxy != this && factory != null) { + factory.updateBackend(proxy, restore); + } else { + JyAttribute.setAttr(formerReferent, JyAttribute.WEAK_REF_ATTR, restore); + } restore.references = references; objects.remove(this); - objects.put(restore, restore); + objects.put(restore, proxy == this ? restore : proxy); AbstractReference aref; for (int i = references.size() - 1; i >= 0; i--) { aref = getReferenceAt(i); if (aref == null) { references.remove(i); } else { - aref.gref = restore; + if (this == proxy) { + aref.gref = restore; + } Thread pendingGet = (Thread) JyAttribute.getAttr( aref, JyAttribute.WEAKREF_PENDING_GET_ATTR); if (pendingGet != null) { @@ -244,7 +315,7 @@ * (The remove from delayed callback list might happen before * the insert.) However we can only set cleared = true after * all gref-variables were updated, otherwise some refs might - * break. To avoid callback-rocessing in the unsafe state + * break. To avoid callback-processing in the unsafe state * between these actions, this method is synchronized (as is call()). */ cleared = true; @@ -278,7 +349,7 @@ * @return an int reference count */ public static int getCount(PyObject object) { - GlobalRef ref = objects.get(new GlobalRef(object)); + ReferenceBackend ref = objects.get(new GlobalRef(object)); return ref == null ? 0 : ref.count(); } @@ -290,7 +361,7 @@ * @return a {@link org.python.core.PyList} of references. May be empty. */ public static PyList getRefs(PyObject object) { - GlobalRef ref = objects.get(new GlobalRef(object)); + ReferenceBackend ref = objects.get(new GlobalRef(object)); return ref == null ? new PyList() : ref.refs(); } @@ -366,8 +437,6 @@ } else { delayedCallback(gr); } - objects.remove(gr); - gr = null; } @Override diff --git a/src/org/python/modules/_weakref/ProxyType.java b/src/org/python/modules/_weakref/ProxyType.java --- a/src/org/python/modules/_weakref/ProxyType.java +++ b/src/org/python/modules/_weakref/ProxyType.java @@ -17,11 +17,11 @@ public static final PyType TYPE = PyType.fromClass(ProxyType.class); - public ProxyType(PyType subType, GlobalRef ref, PyObject callback) { + public ProxyType(PyType subType, ReferenceBackend ref, PyObject callback) { super(subType, ref, callback); } - public ProxyType(GlobalRef ref, PyObject callback) { + public ProxyType(ReferenceBackend ref, PyObject callback) { this(TYPE, ref, callback); } diff --git a/src/org/python/modules/_weakref/ReferenceBackend.java b/src/org/python/modules/_weakref/ReferenceBackend.java new file mode 100644 --- /dev/null +++ b/src/org/python/modules/_weakref/ReferenceBackend.java @@ -0,0 +1,15 @@ +package org.python.modules._weakref; + +import org.python.core.PyObject; +import org.python.core.PyList; + +public interface ReferenceBackend { + public PyObject get(); + public void add(AbstractReference ref); + public boolean isCleared(); + public AbstractReference find(Class cls); + public int pythonHashCode(); + public PyList refs(); + public void restore(PyObject formerReferent); + public int count(); +} diff --git a/src/org/python/modules/_weakref/ReferenceBackendFactory.java b/src/org/python/modules/_weakref/ReferenceBackendFactory.java new file mode 100644 --- /dev/null +++ b/src/org/python/modules/_weakref/ReferenceBackendFactory.java @@ -0,0 +1,13 @@ +package org.python.modules._weakref; + +import org.python.core.PyObject; + +/** + * Reserved for use by JyNI. + */ +public interface ReferenceBackendFactory { + + public ReferenceBackend makeBackend(GlobalRef caller, PyObject referent); + public void notifyClear(ReferenceBackend ref, GlobalRef caller); + public void updateBackend(ReferenceBackend ref, GlobalRef caller); +} diff --git a/src/org/python/modules/_weakref/ReferenceType.java b/src/org/python/modules/_weakref/ReferenceType.java --- a/src/org/python/modules/_weakref/ReferenceType.java +++ b/src/org/python/modules/_weakref/ReferenceType.java @@ -15,11 +15,11 @@ public static final PyType TYPE = PyType.fromClass(ReferenceType.class); - public ReferenceType(PyType subType, GlobalRef gref, PyObject callback) { + public ReferenceType(PyType subType, ReferenceBackend gref, PyObject callback) { super(subType, gref, callback); } - public ReferenceType(GlobalRef gref, PyObject callback) { + public ReferenceType(ReferenceBackend gref, PyObject callback) { this(TYPE, gref, callback); } @@ -33,7 +33,7 @@ callback = null; } - GlobalRef gref = GlobalRef.newInstance(ob); + ReferenceBackend gref = GlobalRef.newInstance(ob); if (new_.for_type == subtype) { // NOTE: CPython disallows weakrefs to many builtin types (e.g. dict, list) // and would check weakrefability here. We aren't as strict since the JVM can diff --git a/src/org/python/modules/_weakref/ReferenceTypeDerived.java b/src/org/python/modules/_weakref/ReferenceTypeDerived.java --- a/src/org/python/modules/_weakref/ReferenceTypeDerived.java +++ b/src/org/python/modules/_weakref/ReferenceTypeDerived.java @@ -73,7 +73,7 @@ dict=new PyStringMap(); } - public ReferenceTypeDerived(PyType subtype,GlobalRef gref,PyObject callback) { + public ReferenceTypeDerived(PyType subtype,ReferenceBackend gref,PyObject callback) { super(subtype,gref,callback); slots=new PyObject[subtype.getNumSlots()]; dict=subtype.instDict(); diff --git a/src/org/python/modules/_weakref/WeakrefModule.java b/src/org/python/modules/_weakref/WeakrefModule.java --- a/src/org/python/modules/_weakref/WeakrefModule.java +++ b/src/org/python/modules/_weakref/WeakrefModule.java @@ -28,7 +28,7 @@ } public static ProxyType proxy(PyObject object) { - GlobalRef gref = GlobalRef.newInstance(object); + ReferenceBackend gref = GlobalRef.newInstance(object); boolean callable = object.isCallable(); ProxyType ret = (ProxyType)gref.find(callable ? CallableProxyType.class : ProxyType.class); if (ret != null) { diff --git a/src/org/python/modules/gc.java b/src/org/python/modules/gc.java --- a/src/org/python/modules/gc.java +++ b/src/org/python/modules/gc.java @@ -25,6 +25,7 @@ import org.python.core.Untraversable; import org.python.core.finalization.FinalizeTrigger; import org.python.modules._weakref.GlobalRef; +import org.python.modules._weakref.ReferenceBackend; //These imports belong to the out-commented section on MXBean-based //gc-sync far below. That section is kept to document this failed @@ -902,7 +903,7 @@ * @see #PRESERVE_WEAKREFS_ON_RESURRECTION */ public static void restoreWeakReferences(PyObject rst) { - GlobalRef toRestore = (GlobalRef) + ReferenceBackend toRestore = (ReferenceBackend) JyAttribute.getAttr(rst, JyAttribute.WEAK_REF_ATTR); if (toRestore != null) { toRestore.restore(rst); diff --git a/src/templates/weakref.derived b/src/templates/weakref.derived --- a/src/templates/weakref.derived +++ b/src/templates/weakref.derived @@ -1,4 +1,4 @@ base_class: ReferenceType want_dict: true -ctr: GlobalRef gref, PyObject callback +ctr: ReferenceBackend gref, PyObject callback incl: object -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Aug 31 21:20:23 2015 From: jython-checkins at python.org (jeff.allen) Date: Mon, 31 Aug 2015 19:20:23 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixes_=231423_so_circular_i?= =?utf-8?q?mports_no_longer_cause_RuntimeError=2E?= Message-ID: <20150831192023.5498.86087@psf.io> https://hg.python.org/jython/rev/b2c98eeaa744 changeset: 7717:b2c98eeaa744 user: Jeff Allen date: Mon Aug 31 19:29:18 2015 +0100 summary: Fixes #1423 so circular imports no longer cause RuntimeError. files: NEWS | 1 + src/org/python/core/PyModule.java | 20 ++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1b1 Bugs fixed + - [ 1423 ] Circular imports no longer cause RuntimeError. - [ 2310 ] test_import runs on Windows (and passes with 4 skips). - [ 2347 ] failures in test_import_pep328 when run with -m - [ 2158, 2259 ] Fixed behaviour of relative from ... import * diff --git a/src/org/python/core/PyModule.java b/src/org/python/core/PyModule.java --- a/src/org/python/core/PyModule.java +++ b/src/org/python/core/PyModule.java @@ -66,24 +66,29 @@ } } + @Override public PyObject fastGetDict() { return __dict__; } + @Override public PyObject getDict() { return __dict__; } + @Override @ExposedSet(name = "__dict__") public void setDict(PyObject newDict) { throw Py.TypeError("readonly attribute"); } + @Override @ExposedDelete(name = "__dict__") public void delDict() { throw Py.TypeError("readonly attribute"); } + @Override protected PyObject impAttr(String name) { if (__dict__ == null) { return null; @@ -96,14 +101,19 @@ if (pyName == null) { return null; } - PyObject attr = null; + String fullName = (pyName.__str__().toString() + '.' + name).intern(); + PyObject modules = Py.getSystemState().modules; + PyObject attr = modules.__finditem__(fullName); + if (path == Py.None) { // XXX: disabled //attr = imp.loadFromClassLoader(fullName, // Py.getSystemState().getClassLoader()); } else if (path instanceof PyList) { - attr = imp.find_module(name, fullName, (PyList)path); + if (attr == null) { + attr = imp.find_module(name, fullName, (PyList)path); + } } else { throw Py.TypeError("__path__ must be list or None"); } @@ -114,7 +124,7 @@ if (attr != null) { // Allow a package component to change its own meaning - PyObject found = Py.getSystemState().modules.__finditem__(fullName); + PyObject found = modules.__finditem__(fullName); if (found != null) { attr = found; } @@ -125,6 +135,7 @@ return null; } + @Override public void __setattr__(String name, PyObject value) { module___setattr__(name, value); } @@ -137,6 +148,7 @@ super.__setattr__(name, value); } + @Override public void __delattr__(String name) { module___delattr__(name); } @@ -146,6 +158,7 @@ super.__delattr__(name); } + @Override public String toString() { return module_toString(); } @@ -168,6 +181,7 @@ return String.format("", name, filename); } + @Override public PyObject __dir__() { // Some special casing to ensure that classes deriving from PyModule // can use their own __dict__. Although it would be nice to do this in -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Aug 31 21:20:23 2015 From: jython-checkins at python.org (jeff.allen) Date: Mon, 31 Aug 2015 19:20:23 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_releases_2=2E7_and_2=2E?= =?utf-8?q?7b1_headings_to_NEWS?= Message-ID: <20150831192022.31177.84903@psf.io> https://hg.python.org/jython/rev/036ec1773838 changeset: 7716:036ec1773838 user: Jeff Allen date: Mon Aug 31 13:12:17 2015 +0100 summary: Add releases 2.7 and 2.7b1 headings to NEWS Do a bit of record-keeping before we lose track. 2.7.1b1 is not tagged, but is the first release to contain the fixes now being added (according to plans). files: NEWS | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ For more details, please see https://hg.python.org/jython +Jython 2.7.1b1 Bugs fixed - [ 2310 ] test_import runs on Windows (and passes with 4 skips). - [ 2347 ] failures in test_import_pep328 when run with -m @@ -9,6 +10,9 @@ - [ 1879 ] -m command now executes scripts from inside a jar file - [ 2058 ] ClasspathPyImporter implements PEP 302 get_data (and others) +Jython 2.7 + same as 2.7rc3 + Jython 2.7rc3 Bugs fixed - [ 2311, 2319 ] Many compatibility fixes for launcher (bin/jython, bin/jython.exe) -- Repository URL: https://hg.python.org/jython