https://github.com/python/cpython/commit/d23f5701ad2a64c7877bb37258420c41735... commit: d23f5701ad2a64c7877bb37258420c417357423e branch: main author: Sam Gross <colesbury@gmail.com> committer: colesbury <colesbury@gmail.com> date: 2025-01-28T19:32:27Z summary: gh-128844: Make `_Py_TryIncref` public as an unstable API. (#128926) This exposes `_Py_TryIncref` as `PyUnstable_TryIncref()` and the helper function `_PyObject_SetMaybeWeakref` as `PyUnstable_EnableTryIncRef`. These are helpers for dealing with unowned references in a safe way, particularly in the free threading build. Co-authored-by: Petr Viktorin <encukou@gmail.com> files: A Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst M Doc/c-api/object.rst M Include/cpython/object.h M Modules/_testcapi/object.c M Objects/object.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 934b2ef06d3108..1ba5942c63601d 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -624,3 +624,84 @@ Object Protocol be immortal in another. .. versionadded:: next + +.. c:function:: int PyUnstable_TryIncRef(PyObject *obj) + + Increments the reference count of *obj* if it is not zero. Returns ``1`` + if the object's reference count was successfully incremented. Otherwise, + this function returns ``0``. + + :c:func:`PyUnstable_EnableTryIncRef` must have been called + earlier on *obj* or this function may spuriously return ``0`` in the + :term:`free threading` build. + + This function is logically equivalent to the following C code, except that + it behaves atomically in the :term:`free threading` build:: + + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; + + This is intended as a building block for managing weak references + without the overhead of a Python :ref:`weak reference object <weakrefobjects>`. + + Typically, correct use of this function requires support from *obj*'s + deallocator (:c:member:`~PyTypeObject.tp_dealloc`). + For example, the following sketch could be adapted to implement a + "weakmap" that works like a :py:class:`~weakref.WeakValueDictionary` + for a specific type: + + .. code-block:: c + + PyMutex mutex; + + PyObject * + add_entry(weakmap_key_type *key, PyObject *value) + { + PyUnstable_EnableTryIncRef(value); + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_add_entry(weakmap, key, value); + PyMutex_Unlock(&mutex); + Py_RETURN_NONE; + } + + PyObject * + get_value(weakmap_key_type *key) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + PyObject *result = weakmap_find(weakmap, key); + if (PyUnstable_TryIncRef(result)) { + // `result` is safe to use + PyMutex_Unlock(&mutex); + return result; + } + // if we get here, `result` is starting to be garbage-collected, + // but has not been removed from the weakmap yet + PyMutex_Unlock(&mutex); + return NULL; + } + + // tp_dealloc function for weakmap values + void + value_dealloc(PyObject *value) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_remove_value(weakmap, value); + + ... + PyMutex_Unlock(&mutex); + } + + .. versionadded:: 3.14 + +.. c:function:: void PyUnstable_EnableTryIncRef(PyObject *obj) + + Enables subsequent uses of :c:func:`PyUnstable_TryIncRef` on *obj*. The + caller must hold a :term:`strong reference` to *obj* when calling this. + + .. versionadded:: 3.14 diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 4c9e4f6c6e0434..71bd01884426ad 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -544,3 +544,9 @@ PyAPI_FUNC(int) PyUnstable_Object_EnableDeferredRefcount(PyObject *); /* Check whether the object is immortal. This cannot fail. */ PyAPI_FUNC(int) PyUnstable_IsImmortal(PyObject *); + +// Increments the reference count of the object, if it's not zero. +// PyUnstable_EnableTryIncRef() should be called on the object +// before calling this function in order to avoid spurious failures. +PyAPI_FUNC(int) PyUnstable_TryIncRef(PyObject *); +PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *); diff --git a/Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst b/Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst new file mode 100644 index 00000000000000..d9e1962631026a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyUnstable_TryIncRef` and :c:func:`PyUnstable_EnableTryIncRef` +unstable APIs. These are helpers for dealing with unowned references in +a thread-safe way, particularly in the free threading build. diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 1d0169b2af9469..409b0c83c18cfd 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -131,6 +131,59 @@ pyobject_enable_deferred_refcount(PyObject *self, PyObject *obj) return PyLong_FromLong(result); } +static int MyObject_dealloc_called = 0; + +static void +MyObject_dealloc(PyObject *op) +{ + // PyUnstable_TryIncRef should return 0 if object is being deallocated + assert(Py_REFCNT(op) == 0); + assert(!PyUnstable_TryIncRef(op)); + assert(Py_REFCNT(op) == 0); + + MyObject_dealloc_called++; + Py_TYPE(op)->tp_free(op); +} + +static PyTypeObject MyType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "MyType", + .tp_basicsize = sizeof(PyObject), + .tp_dealloc = MyObject_dealloc, +}; + +static PyObject * +test_py_try_inc_ref(PyObject *self, PyObject *unused) +{ + if (PyType_Ready(&MyType) < 0) { + return NULL; + } + + MyObject_dealloc_called = 0; + + PyObject *op = PyObject_New(PyObject, &MyType); + if (op == NULL) { + return NULL; + } + + PyUnstable_EnableTryIncRef(op); +#ifdef Py_GIL_DISABLED + // PyUnstable_EnableTryIncRef sets the shared flags to + // `_Py_REF_MAYBE_WEAKREF` if the flags are currently zero to ensure that + // the shared reference count is merged on deallocation. + assert((op->ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) >= _Py_REF_MAYBE_WEAKREF); +#endif + + if (!PyUnstable_TryIncRef(op)) { + PyErr_SetString(PyExc_AssertionError, "PyUnstable_TryIncRef failed"); + Py_DECREF(op); + return NULL; + } + Py_DECREF(op); // undo try-incref + Py_DECREF(op); // dealloc + assert(MyObject_dealloc_called == 1); + Py_RETURN_NONE; +} static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, @@ -139,6 +192,7 @@ static PyMethodDef test_methods[] = { {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS}, {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O}, {"pyobject_enable_deferred_refcount", pyobject_enable_deferred_refcount, METH_O}, + {"test_py_try_inc_ref", test_py_try_inc_ref, METH_NOARGS}, {NULL}, }; diff --git a/Objects/object.c b/Objects/object.c index eb1a7825c45450..a70a2c3fc2f3dd 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2588,6 +2588,20 @@ PyUnstable_Object_EnableDeferredRefcount(PyObject *op) #endif } +int +PyUnstable_TryIncRef(PyObject *op) +{ + return _Py_TryIncref(op); +} + +void +PyUnstable_EnableTryIncRef(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + _PyObject_SetMaybeWeakref(op); +#endif +} + void _Py_ResurrectReference(PyObject *op) { diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 1aabe262eac480..fbb84fc7950fae 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -447,6 +447,8 @@ Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - Modules/_testcapi/mem.c - FmHook - +Modules/_testcapi/object.c - MyObject_dealloc_called - +Modules/_testcapi/object.c - MyType - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed -