Mailman 3 October 2023 - Python-checkins

gh-111531: Tkinter: fix reference leaks in bind_class() and bind_all() (GH-111533)
by serhiy-storchaka Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/e3353c498d79f0f3f108a9baf8807a12e7… commit: e3353c498d79f0f3f108a9baf8807a12e77c2ebe branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2023-10-31T08:48:52+02:00 summary: gh-111531: Tkinter: fix reference leaks in bind_class() and bind_all() (GH-111533) files: A Misc/NEWS.d/next/Library/2023-10-31-07-46-56.gh-issue-111531.6zUV_G.rst M Lib/tkinter/__init__.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 47b93d31fd3ff..0df7f9d889413 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1538,7 +1538,7 @@ def bind_all(self, sequence=None, func=None, add=None): An additional boolean parameter ADD specifies whether FUNC will be called additionally to the other bound function or whether it will replace the previous function. See bind for the return value.""" - return self._bind(('bind', 'all'), sequence, func, add, 0) + return self._root()._bind(('bind', 'all'), sequence, func, add, True) def unbind_all(self, sequence): """Unbind for all widgets for event SEQUENCE all functions.""" @@ -1552,7 +1552,7 @@ def bind_class(self, className, sequence=None, func=None, add=None): whether it will replace the previous function. See bind for the return value.""" - return self._bind(('bind', className), sequence, func, add, 0) + return self._root()._bind(('bind', className), sequence, func, add, True) def unbind_class(self, className, sequence): """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE diff --git a/Misc/NEWS.d/next/Library/2023-10-31-07-46-56.gh-issue-111531.6zUV_G.rst b/Misc/NEWS.d/next/Library/2023-10-31-07-46-56.gh-issue-111531.6zUV_G.rst new file mode 100644 index 0000000000000..b722f0414184b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-31-07-46-56.gh-issue-111531.6zUV_G.rst @@ -0,0 +1,2 @@ +Fix reference leaks in ``bind_class()`` and ``bind_all()`` methods of +:mod:`tkinter` widgets.

1 0

gh-111301: Move importlib.resources changes to the Removed section (#111509)
by hugovk Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/3dbaed3caa00062087a848740b6e713ad5… commit: 3dbaed3caa00062087a848740b6e713ad55b0aed branch: main author: Karolina Surma <33810531+befeleme(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-10-31T06:46:20Z summary: gh-111301: Move importlib.resources changes to the Removed section (#111509) files: M Doc/whatsnew/3.13.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index eaa9ab5a04c88..3a2df4fc8a3de 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -604,20 +604,6 @@ although there is currently no date scheduled for their removal. * ``EntryPoints`` tuple interface. * Implicit ``None`` on return values. -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use ``files()`` instead. Refer to `importlib-resources: Migrating from Legacy - <https://importlib-resources.readthedocs.io/en/latest/using.html#migrating-f…>`_ - for migration advice. - * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. @@ -830,6 +816,24 @@ configparser and with a deprecation warning since Python 3.11. (Contributed by Hugo van Kemenade in :gh:`104886`.) +importlib +--------- + +* Remove :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy + <https://importlib-resources.readthedocs.io/en/latest/using.html#migrating-f…>`_ + for migration advice. + (Contributed by Jason R. Coombs in :gh:`106532`.) + locale ------

1 0

gh-111301: Move `importlib.resources.files` change to What's new in Python 3.12 (#111512)
by hugovk Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/997683435693876e687cf09d3553abde18… commit: 997683435693876e687cf09d3553abde18787d2e branch: main author: Karolina Surma <33810531+befeleme(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-10-31T06:01:00Z summary: gh-111301: Move `importlib.resources.files` change to What's new in Python 3.12 (#111512) files: M Doc/whatsnew/3.12.rst M Doc/whatsnew/3.13.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 36d12feebea79..72277a023b3b2 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -716,6 +716,9 @@ importlib.resources * :func:`importlib.resources.as_file` now supports resource directories. (Contributed by Jason R. Coombs in :gh:`97930`.) +* Rename first parameter of :func:`importlib.resources.files` to *anchor*. + (Contributed by Jason R. Coombs in :gh:`100598`.) + inspect ------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 84d5daae6e623..eaa9ab5a04c88 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -604,7 +604,6 @@ although there is currently no date scheduled for their removal. * ``EntryPoints`` tuple interface. * Implicit ``None`` on return values. -* :mod:`importlib.resources`: First parameter to files is renamed to 'anchor'. * :mod:`importlib.resources` deprecated methods: * ``contents()``

1 0

gh-111062 CI: Update skip lists for resuable workflows (#111521)
by hugovk Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/173d13bb1f51a3022e48ce98f12b042f7e… commit: 173d13bb1f51a3022e48ce98f12b042f7e8e73ec branch: main author: Hugo van Kemenade <hugovk(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-10-31T07:49:15+02:00 summary: gh-111062 CI: Update skip lists for resuable workflows (#111521) CI: Update skip lists files: M .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0febcedad4411..2d4197d4a2982 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -500,11 +500,13 @@ jobs: - check_source # Transitive dependency, needed to access `run_tests` value - check-docs - check_generated_files - - build_windows - build_macos + - build_macos_free_threaded - build_ubuntu - build_ubuntu_free_threaded - build_ubuntu_ssltests + - build_windows + - build_windows_free_threaded - test_hypothesis - build_asan - cifuzz @@ -517,10 +519,10 @@ jobs: with: allowed-failures: >- build_macos, + build_macos_free_threaded, build_ubuntu_free_threaded, build_ubuntu_ssltests, - build_win32, - build_win_arm64, + build_windows_free_threaded, cifuzz, test_hypothesis, allowed-skips: >- @@ -535,13 +537,13 @@ jobs: needs.check_source.outputs.run_tests != 'true' && ' check_generated_files, - build_win32, - build_win_amd64, - build_win_arm64, build_macos, + build_macos_free_threaded, build_ubuntu, build_ubuntu_free_threaded, build_ubuntu_ssltests, + build_windows, + build_windows_free_threaded, build_asan, ' || ''

1 0

[3.11] Remove myself from typing CODEOWNERS (GH-111523) (#111526)
by hugovk Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/c5f6c6396d15549c1e68094ed9811ae431… commit: c5f6c6396d15549c1e68094ed9811ae4316863f5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-10-31T07:47:57+02:00 summary: [3.11] Remove myself from typing CODEOWNERS (GH-111523) (#111526) Remove myself from typing CODEOWNERS (GH-111523) (cherry picked from commit 804a207c168b876112984709edc3a94afa433c69) Co-authored-by: Ken Jin <kenjin(a)python.org> files: M .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1b156c829cd9a..6c29ba321b923 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -137,7 +137,7 @@ Lib/ast.py @isidentical **/*idlelib* @terryjreedy -**/*typing* @gvanrossum @Fidget-Spinner @JelleZijlstra @AlexWaygood +**/*typing* @gvanrossum @JelleZijlstra @AlexWaygood **/*asyncore @giampaolo **/*asynchat @giampaolo

1 0

[3.11] gh-111347: Remove wrong assertion in test_sendfile (GH-111377) (#111462)
by willingc Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/a9d5966bce9bb347002d4da7672bf49fe4… commit: a9d5966bce9bb347002d4da7672bf49fe4274ea4 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: willingc <carolcode(a)willingconsulting.com> date: 2023-10-30T18:14:30-07:00 summary: [3.11] gh-111347: Remove wrong assertion in test_sendfile (GH-111377) (#111462) gh-111347: Remove wrong assertion in test_sendfile (GH-111377) Windows is different. (cherry picked from commit fa35b9e89b2e207fc8bae9eb0284260d0d922e7a) Co-authored-by: zcxsythenew <30565051+zcxsythenew(a)users.noreply.github.com> files: M Lib/test/test_asyncio/test_sendfile.py diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index 0198da21d7702..d33ff197bbfa1 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -470,8 +470,11 @@ def test_sendfile_close_peer_in_the_middle_of_receiving(self): self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), srv_proto.nbytes) - self.assertTrue(1024 <= self.file.tell() < len(self.DATA), - self.file.tell()) + if not (sys.platform == 'win32' + and isinstance(self.loop, asyncio.ProactorEventLoop)): + # On Windows, Proactor uses transmitFile, which does not update tell() + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) self.assertTrue(cli_proto.transport.is_closing()) def test_sendfile_fallback_close_peer_in_the_middle_of_receiving(self):

1 0

gh-111277: In summarize_stats.py, don't fail fast on invalid ratios (#111278)
by gvanrossum Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/9495bcaf599e0961b45447b8fab7f84469… commit: 9495bcaf599e0961b45447b8fab7f8446945ba1a branch: main author: Michael Droettboom <mdboom(a)gmail.com> committer: gvanrossum <gvanrossum(a)gmail.com> date: 2023-10-30T17:10:07-07:00 summary: gh-111277: In summarize_stats.py, don't fail fast on invalid ratios (#111278) files: M Tools/scripts/summarize_stats.py diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 0527d4bebfb86..360b7c720bd1f 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -421,8 +421,6 @@ def __init__(self, num: int, den: int | None, percentage: bool = True): self.num = num self.den = den self.percentage = percentage - if den == 0 and num != 0: - raise ValueError("Invalid denominator") def __float__(self): if self.den == 0: @@ -433,7 +431,11 @@ def __float__(self): return self.num / self.den def markdown(self) -> str: - if self.den == 0 or self.den is None: + if self.den is None: + return "" + elif self.den == 0: + if self.num != 0: + return f"{self.num:,} / 0 !!" return "" elif self.percentage: return f"{self.num / self.den:,.01%}"

1 0

gh-109329: Count tier2 opcode misses (#110561)
by gvanrossum Oct. 31, 2023

Oct. 31, 2023

https://github.com/python/cpython/commit/84b4533e8446cbff3325fffe939c87f712… commit: 84b4533e8446cbff3325fffe939c87f7120a3ffd branch: main author: Michael Droettboom <mdboom(a)gmail.com> committer: gvanrossum <gvanrossum(a)gmail.com> date: 2023-10-30T17:02:45-07:00 summary: gh-109329: Count tier2 opcode misses (#110561) This keeps a separate 'miss' counter for each micro-opcode, incremented whenever a guard uop takes a deoptimization side exit. files: M Include/cpython/pystats.h M Include/internal/pycore_code.h M Python/executor.c M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 4988caa803723..294bf1505f011 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -98,6 +98,7 @@ typedef struct _gc_stats { typedef struct _uop_stats { uint64_t execution_count; + uint64_t miss; } UOpStats; #define _Py_UOP_HIST_SIZE 32 diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index d31d8363d771c..4f126abf47644 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -283,7 +283,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) -#define UOP_EXE_INC(opname) do { if (_Py_stats) _Py_stats->optimization_stats.opcode[opname].execution_count++; } while (0) +#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) #define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) #define OPT_HIST(length, name) \ do { \ @@ -308,7 +308,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) #define GC_STAT_ADD(gen, name, n) ((void)0) #define OPT_STAT_INC(name) ((void)0) -#define UOP_EXE_INC(opname) ((void)0) +#define UOP_STAT_INC(opname, name) ((void)0) #define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) #endif // !Py_STATS diff --git a/Python/executor.c b/Python/executor.c index bfa7f7e1c3d84..2884565689171 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -25,6 +25,7 @@ #undef DEOPT_IF #define DEOPT_IF(COND, INSTNAME) \ if ((COND)) { \ + UOP_STAT_INC(INSTNAME, miss); \ goto deoptimize; \ } @@ -93,7 +94,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject (int)(stack_pointer - _PyFrame_Stackbase(frame))); pc++; OPT_STAT_INC(uops_executed); - UOP_EXE_INC(opcode); + UOP_STAT_INC(opcode, execution_count); #ifdef Py_STATS trace_uop_execution_counter++; #endif diff --git a/Python/specialize.c b/Python/specialize.c index d74c4c58a9452..41e74c67d8c9a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -248,6 +248,9 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) if (stats->opcode[i].execution_count) { fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", names[i], stats->opcode[i].execution_count); } + if (stats->opcode[i].miss) { + fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", names[i], stats->opcode[i].miss); + } } for (int i = 0; i < 256; i++) { diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 165b9b40566a2..0527d4bebfb86 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -1028,7 +1028,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ], ) yield Section( - "Uop stats", + "Uop execution stats", "", [ Table(

1 0

gh-76785: Move the Cross-Interpreter Code to Its Own File (gh-111502)
by ericsnowcurrently Oct. 30, 2023

Oct. 30, 2023

https://github.com/python/cpython/commit/c6fe0869ab1d91525f88279f8567461082… commit: c6fe0869ab1d91525f88279f8567461082c0d3ce branch: main author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: ericsnowcurrently <ericsnowcurrently(a)gmail.com> date: 2023-10-30T16:53:10-06:00 summary: gh-76785: Move the Cross-Interpreter Code to Its Own File (gh-111502) This is partly to clear this stuff out of pystate.c, but also in preparation for moving some code out of _xxsubinterpretersmodule.c. This change also moves this stuff to the internal API (new: Include/internal/pycore_crossinterp.h). @vstinner did this previously and I undid it. Now I'm re-doing it. :/ files: A Include/internal/pycore_crossinterp.h A Python/crossinterp.c M Include/cpython/pystate.h M Include/internal/pycore_ceval.h M Include/internal/pycore_interp.h M Include/internal/pycore_runtime.h M Lib/test/test__xxsubinterpreters.py M Makefile.pre.in M Modules/_testcapimodule.c M Modules/_testinternalcapi.c M Modules/_xxinterpchannelsmodule.c M Modules/_xxsubinterpretersmodule.c M Objects/abstract.c M PCbuild/_freeze_module.vcxproj M PCbuild/_freeze_module.vcxproj.filters M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/pystate.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 995f02eab5863..ec99f90d669d1 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -258,80 +258,3 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); - - -/* cross-interpreter data */ - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid _PyCrossInterpreterData; - -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); -typedef void (*xid_freefunc)(void *); - -struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. Note that IDs are never re-used, so - // each one will always correspond to a specific interpreter - // (whether still alive or not). - int64_t interpid; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - xid_newobjectfunc new_object; - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - xid_freefunc free; -}; - -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, - PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, - PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -/* cross-interpreter data registry */ - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index d8afee90a32e2..339ced3c87a43 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -54,16 +54,6 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall( void *arg, int flags); -typedef int (*_Py_simple_func)(void *); -extern int _Py_CallInInterpreter( - PyInterpreterState *interp, - _Py_simple_func func, - void *arg); -extern int _Py_CallInInterpreterAndRawFree( - PyInterpreterState *interp, - _Py_simple_func func, - void *arg); - extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h new file mode 100644 index 0000000000000..59e4cd9ece780 --- /dev/null +++ b/Include/internal/pycore_crossinterp.h @@ -0,0 +1,139 @@ +#ifndef Py_INTERNAL_CROSSINTERP_H +#define Py_INTERNAL_CROSSINTERP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/***************************/ +/* cross-interpreter calls */ +/***************************/ + +typedef int (*_Py_simple_func)(void *); +extern int _Py_CallInInterpreter( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); +extern int _Py_CallInInterpreterAndRawFree( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); + + +/**************************/ +/* cross-interpreter data */ +/**************************/ + +typedef struct _xid _PyCrossInterpreterData; +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass safely between interpreters in the same process. +struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case the code that sets the field, + // likely a registered "crossinterpdatafunc", is responsible for + // ensuring it owns the reference (i.e. incref). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. Note that IDs are never re-used, so + // each one will always correspond to a specific interpreter + // (whether still alive or not). + int64_t interpid; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + xid_newobjectfunc new_object; + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + xid_freefunc free; +}; + +PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); +PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); + + +/* defining cross-interpreter data */ + +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + + +/* using cross-interpreter data */ + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); + + +/* cross-interpreter data registry */ + +// For now we use a global registry of shareable classes. An +// alternative would be to add a tp_* slot for a class's +// crossinterpdatafunc. It would be simpler and more efficient. + +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); + +struct _xidregitem; + +struct _xidregitem { + struct _xidregitem *prev; + struct _xidregitem *next; + /* This can be a dangling pointer, but only if weakref is set. */ + PyTypeObject *cls; + /* This is NULL for builtin types. */ + PyObject *weakref; + size_t refcount; + crossinterpdatafunc getdata; +}; + +struct _xidregistry { + PyThread_type_lock mutex; + struct _xidregitem *head; +}; + +PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CROSSINTERP_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index fc27aad48b583..a067a60eca05d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -15,6 +15,7 @@ extern "C" { #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache #include "pycore_context.h" // struct _Py_context_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_dict_state.h" // struct _Py_dict_state #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state @@ -41,28 +42,6 @@ struct _Py_long_state { /* cross-interpreter data registry */ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -struct _xidregitem; - -struct _xidregitem { - struct _xidregitem *prev; - struct _xidregitem *next; - /* This can be a dangling pointer, but only if weakref is set. */ - PyTypeObject *cls; - /* This is NULL for builtin types. */ - PyObject *weakref; - size_t refcount; - crossinterpdatafunc getdata; -}; - -struct _xidregistry { - PyThread_type_lock mutex; - struct _xidregitem *head; -}; - /* interpreter state */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index ec026d42f6d10..320e5bbedc068 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_atexit.h" // struct _atexit_runtime_state #include "pycore_ceval_state.h" // struct _ceval_runtime_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index e3c917aa2eb19..f0b9101f0a20b 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -7,7 +7,7 @@ import threading import unittest -import _testcapi +import _testinternalcapi from test import support from test.support import import_helper from test.support import script_helper @@ -146,8 +146,8 @@ class ShareableTypeTests(unittest.TestCase): def _assert_values(self, values): for obj in values: with self.subTest(obj): - xid = _testcapi.get_crossinterp_data(obj) - got = _testcapi.restore_crossinterp_data(xid) + xid = _testinternalcapi.get_crossinterp_data(obj) + got = _testinternalcapi.restore_crossinterp_data(xid) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -155,8 +155,8 @@ def _assert_values(self, values): def test_singletons(self): for obj in [None]: with self.subTest(obj): - xid = _testcapi.get_crossinterp_data(obj) - got = _testcapi.restore_crossinterp_data(xid) + xid = _testinternalcapi.get_crossinterp_data(obj) + got = _testinternalcapi.restore_crossinterp_data(xid) # XXX What about between interpreters? self.assertIs(got, obj) @@ -187,7 +187,7 @@ def test_non_shareable_int(self): for i in ints: with self.subTest(i): with self.assertRaises(OverflowError): - _testcapi.get_crossinterp_data(i) + _testinternalcapi.get_crossinterp_data(i) class ModuleTests(TestBase): diff --git a/Makefile.pre.in b/Makefile.pre.in index 733c68542ee88..f2b252cce9775 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -409,6 +409,7 @@ PYTHON_OBJS= \ Python/codecs.o \ Python/compile.o \ Python/context.o \ + Python/crossinterp.o \ Python/dynamic_annotations.o \ Python/errors.o \ Python/executor.o \ @@ -1800,6 +1801,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_complexobject.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_crossinterp.h \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dict_state.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index dc9a25b6c9ff3..fc9dc746095b7 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1445,58 +1445,6 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } -static void -_xid_capsule_destructor(PyObject *capsule) -{ - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); - if (data != NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - PyMem_Free(data); - } -} - -static PyObject * -get_crossinterp_data(PyObject *self, PyObject *args) -{ - PyObject *obj = NULL; - if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { - return NULL; - } - - _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); - if (data == NULL) { - PyErr_NoMemory(); - return NULL; - } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { - PyMem_Free(data); - return NULL; - } - PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); - if (capsule == NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - PyMem_Free(data); - } - return capsule; -} - -static PyObject * -restore_crossinterp_data(PyObject *self, PyObject *args) -{ - PyObject *capsule = NULL; - if (!PyArg_ParseTuple(args, "O:restore_crossinterp_data", &capsule)) { - return NULL; - } - - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); - if (data == NULL) { - return NULL; - } - return _PyCrossInterpreterData_NewObject(data); -} - static PyMethodDef ml; static PyObject * @@ -3282,8 +3230,6 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, - {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, - {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1869f48c2b1fb..a71e7e1dcc125 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1466,6 +1466,58 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } +static void +_xid_capsule_destructor(PyObject *capsule) +{ + _PyCrossInterpreterData *data = \ + (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + if (data != NULL) { + assert(_PyCrossInterpreterData_Release(data) == 0); + _PyCrossInterpreterData_Free(data); + } +} + +static PyObject * +get_crossinterp_data(PyObject *self, PyObject *args) +{ + PyObject *obj = NULL; + if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { + return NULL; + } + + _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + if (data == NULL) { + return NULL; + } + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + _PyCrossInterpreterData_Free(data); + return NULL; + } + PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); + if (capsule == NULL) { + assert(_PyCrossInterpreterData_Release(data) == 0); + _PyCrossInterpreterData_Free(data); + } + return capsule; +} + +static PyObject * +restore_crossinterp_data(PyObject *self, PyObject *args) +{ + PyObject *capsule = NULL; + if (!PyArg_ParseTuple(args, "O:restore_crossinterp_data", &capsule)) { + return NULL; + } + + _PyCrossInterpreterData *data = \ + (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + if (data == NULL) { + return NULL; + } + return _PyCrossInterpreterData_NewObject(data); +} + + /*[clinic input] _testinternalcapi.write_unraisable_exc exception as exc: object @@ -1645,6 +1697,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, + {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, + {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_WRITE_UNRAISABLE_EXC_METHODDEF _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {NULL, NULL} /* sentinel */ diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 11fe8cd01fc47..1c9ae3b87adf7 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -7,6 +7,7 @@ #include "Python.h" #include "interpreteridobject.h" +#include "pycore_crossinterp.h" // struct _xid #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_interp.h" // _PyInterpreterState_LookUpID() diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 69078dc74d5c3..640fd69061d92 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -6,6 +6,7 @@ #endif #include "Python.h" +#include "pycore_crossinterp.h" // struct _xid #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() diff --git a/Objects/abstract.c b/Objects/abstract.c index 806ca6584bda9..070e762c58a19 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -5,6 +5,7 @@ #include "pycore_pybuffer.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() +#include "pycore_crossinterp.h" // _Py_CallInInterpreter() #include "pycore_object.h" // _Py_CheckSlotResult() #include "pycore_long.h" // _Py_IsNegative #include "pycore_pyerrors.h" // _PyErr_Occurred() diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index cfcb6cb433352..05b8bfdc38a99 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -209,6 +209,7 @@ <ClCompile Include="..\Python\codecs.c" /> <ClCompile Include="..\Python\compile.c" /> <ClCompile Include="..\Python\context.c" /> + <ClCompile Include="..\Python\crossinterp.c" /> <ClCompile Include="..\Python\dtoa.c" /> <ClCompile Include="..\Python\dynamic_annotations.c" /> <ClCompile Include="..\Python\dynload_win.c" /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index f0afa7a6f53e5..d6cbd2d3d4736 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -103,6 +103,9 @@ <ClCompile Include="..\Python\context.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\crossinterp.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\descrobject.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 203d9c858173d..954a59a0bc701 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -216,6 +216,7 @@ <ClInclude Include="..\Include\internal\pycore_complexobject.h" /> <ClInclude Include="..\Include\internal\pycore_condvar.h" /> <ClInclude Include="..\Include\internal\pycore_context.h" /> + <ClInclude Include="..\Include\internal\pycore_crossinterp.h" /> <ClInclude Include="..\Include\internal\pycore_descrobject.h" /> <ClInclude Include="..\Include\internal\pycore_dict.h" /> <ClInclude Include="..\Include\internal\pycore_dict_state.h" /> @@ -561,6 +562,7 @@ <ClCompile Include="..\Python\codecs.c" /> <ClCompile Include="..\Python\compile.c" /> <ClCompile Include="..\Python\context.c" /> + <ClCompile Include="..\Python\crossinterp.c" /> <ClCompile Include="..\Python\dynamic_annotations.c" /> <ClCompile Include="..\Python\dynload_win.c" /> <ClCompile Include="..\Python\errors.c" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 09f4a2f27cdc0..2f8b206f973f3 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -579,6 +579,9 @@ <ClInclude Include="..\Include\internal\pycore_context.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_crossinterp.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_descrobject.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -1280,6 +1283,9 @@ <ClCompile Include="..\Python\compile.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\crossinterp.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Python\dynamic_annotations.c"> <Filter>Python</Filter> </ClCompile> diff --git a/Python/crossinterp.c b/Python/crossinterp.c new file mode 100644 index 0000000000000..74f1d6ecef132 --- /dev/null +++ b/Python/crossinterp.c @@ -0,0 +1,627 @@ + +/* API for managing interactions between isolated interpreters */ + +#include "Python.h" +#include "pycore_ceval.h" // _Py_simple_func +#include "pycore_crossinterp.h" // struct _xid +#include "pycore_pyerrors.h" // _PyErr_Clear() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_weakref.h" // _PyWeakref_GET_REF() + + +/***************************/ +/* cross-interpreter calls */ +/***************************/ + +int +_Py_CallInInterpreter(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == _PyThreadState_GetCurrent()->interp) { + return func(arg); + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); + return 0; +} + +int +_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == _PyThreadState_GetCurrent()->interp) { + int res = func(arg); + PyMem_RawFree(arg); + return res; + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); + return 0; +} + + +/**************************/ +/* cross-interpreter data */ +/**************************/ + +_PyCrossInterpreterData * +_PyCrossInterpreterData_New(void) +{ + _PyCrossInterpreterData *xid = PyMem_RawMalloc( + sizeof(_PyCrossInterpreterData)); + if (xid == NULL) { + PyErr_NoMemory(); + } + return xid; +} + +void +_PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyCrossInterpreterData_Clear(interp, xid); + PyMem_RawFree(xid); +} + + +/* defining cross-interpreter data */ + +static inline void +_xidata_init(_PyCrossInterpreterData *data) +{ + // If the value is being reused + // then _xidata_clear() should have been called already. + assert(data->data == NULL); + assert(data->obj == NULL); + *data = (_PyCrossInterpreterData){0}; + data->interpid = -1; +} + +static inline void +_xidata_clear(_PyCrossInterpreterData *data) +{ + // _PyCrossInterpreterData only has two members that need to be + // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. + // In both cases the original (owning) interpreter must be used, + // which is the caller's responsibility to ensure. + if (data->data != NULL) { + if (data->free != NULL) { + data->free(data->data); + } + data->data = NULL; + } + Py_CLEAR(data->obj); +} + +void +_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(data != NULL); + assert(new_object != NULL); + _xidata_init(data); + data->data = shared; + if (obj != NULL) { + assert(interp != NULL); + // released in _PyCrossInterpreterData_Clear() + data->obj = Py_NewRef(obj); + } + // Ideally every object would know its owning interpreter. + // Until then, we have to rely on the caller to identify it + // (but we don't need it in all cases). + data->interpid = (interp != NULL) ? interp->id : -1; + data->new_object = new_object; +} + +int +_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(size > 0); + // For now we always free the shared data in the same interpreter + // where it was allocated, so the interpreter is required. + assert(interp != NULL); + _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + data->data = PyMem_RawMalloc(size); + if (data->data == NULL) { + return -1; + } + data->free = PyMem_RawFree; + return 0; +} + +void +_PyCrossInterpreterData_Clear(PyInterpreterState *interp, + _PyCrossInterpreterData *data) +{ + assert(data != NULL); + // This must be called in the owning interpreter. + assert(interp == NULL + || data->interpid == -1 + || data->interpid == interp->id); + _xidata_clear(data); +} + + +/* using cross-interpreter data */ + +static int +_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) +{ + // data->data can be anything, including NULL, so we don't check it. + + // data->obj may be NULL, so we don't check it. + + if (data->interpid < 0) { + _PyErr_SetString(tstate, PyExc_SystemError, "missing interp"); + return -1; + } + + if (data->new_object == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "missing new_object func"); + return -1; + } + + // data->free may be NULL, so we don't check it. + + return 0; +} + +crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); + +/* This is a separate func from _PyCrossInterpreterData_Lookup in order + to keep the registry code separate. */ +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) +{ + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S does not support cross-interpreter data", obj); + return getdata; +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + return -1; + } + return 0; +} + +int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + PyThreadState *tstate = _PyThreadState_GetCurrent(); +#ifdef Py_DEBUG + // The caller must hold the GIL + _Py_EnsureTstateNotNULL(tstate); +#endif + PyInterpreterState *interp = tstate->interp; + + // Reset data before re-populating. + *data = (_PyCrossInterpreterData){0}; + data->interpid = -1; + + // Call the "getdata" func for the object. + Py_INCREF(obj); + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + Py_DECREF(obj); + return -1; + } + int res = getdata(tstate, obj, data); + Py_DECREF(obj); + if (res != 0) { + return -1; + } + + // Fill in the blanks and validate the result. + data->interpid = interp->id; + if (_check_xidata(tstate, data) != 0) { + (void)_PyCrossInterpreterData_Release(data); + return -1; + } + + return 0; +} + +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); +} + +static int +_call_clear_xidata(void *data) +{ + _xidata_clear((_PyCrossInterpreterData *)data); + return 0; +} + +static int +_xidata_release(_PyCrossInterpreterData *data, int rawfree) +{ + if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { + // Nothing to release! + if (rawfree) { + PyMem_RawFree(data); + } + else { + data->data = NULL; + } + return 0; + } + + // Switch to the original interpreter. + PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interpid); + if (interp == NULL) { + // The interpreter was already destroyed. + // This function shouldn't have been called. + // XXX Someone leaked some memory... + assert(PyErr_Occurred()); + if (rawfree) { + PyMem_RawFree(data); + } + return -1; + } + + // "Release" the data and/or the object. + if (rawfree) { + return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, data); + } + else { + return _Py_CallInInterpreter(interp, _call_clear_xidata, data); + } +} + +int +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 0); +} + +int +_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 1); +} + + +/* registry of {type -> crossinterpdatafunc} */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +static int +_xidregistry_add_type(struct _xidregistry *xidregistry, + PyTypeObject *cls, crossinterpdatafunc getdata) +{ + struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + if (newhead == NULL) { + return -1; + } + *newhead = (struct _xidregitem){ + // We do not keep a reference, to avoid keeping the class alive. + .cls = cls, + .refcount = 1, + .getdata = getdata, + }; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // XXX Assign a callback to clear the entry from the registry? + newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->weakref == NULL) { + PyMem_RawFree(newhead); + return -1; + } + } + newhead->next = xidregistry->head; + if (newhead->next != NULL) { + newhead->next->prev = newhead; + } + xidregistry->head = newhead; + return 0; +} + +static struct _xidregitem * +_xidregistry_remove_entry(struct _xidregistry *xidregistry, + struct _xidregitem *entry) +{ + struct _xidregitem *next = entry->next; + if (entry->prev != NULL) { + assert(entry->prev->next == entry); + entry->prev->next = next; + } + else { + assert(xidregistry->head == entry); + xidregistry->head = next; + } + if (next != NULL) { + next->prev = entry->prev; + } + Py_XDECREF(entry->weakref); + PyMem_RawFree(entry); + return next; +} + +// This is used in pystate.c (for now). +void +_Py_xidregistry_clear(struct _xidregistry *xidregistry) +{ + struct _xidregitem *cur = xidregistry->head; + xidregistry->head = NULL; + while (cur != NULL) { + struct _xidregitem *next = cur->next; + Py_XDECREF(cur->weakref); + PyMem_RawFree(cur); + cur = next; + } +} + +static struct _xidregitem * +_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +{ + struct _xidregitem *cur = xidregistry->head; + while (cur != NULL) { + if (cur->weakref != NULL) { + // cur is/was a heap type. + PyObject *registered = _PyWeakref_GET_REF(cur->weakref); + if (registered == NULL) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + continue; + } + assert(PyType_Check(registered)); + assert(cur->cls == (PyTypeObject *)registered); + assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + Py_DECREF(registered); + } + if (cur->cls == cls) { + return cur; + } + cur = cur->next; + } + return NULL; +} + +static inline struct _xidregistry * +_get_xidregistry(PyInterpreterState *interp, PyTypeObject *cls) +{ + struct _xidregistry *xidregistry = &interp->runtime->xidregistry; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + assert(interp->xidregistry.mutex == xidregistry->mutex); + xidregistry = &interp->xidregistry; + } + return xidregistry; +} + +static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); + +static inline void +_ensure_builtins_xid(PyInterpreterState *interp, struct _xidregistry *xidregistry) +{ + if (xidregistry != &interp->xidregistry) { + assert(xidregistry == &interp->runtime->xidregistry); + if (xidregistry->head == NULL) { + _register_builtins_for_crossinterpreter_data(xidregistry); + } + } +} + +int +_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, + crossinterpdatafunc getdata) +{ + if (!PyType_Check(cls)) { + PyErr_Format(PyExc_ValueError, "only classes may be registered"); + return -1; + } + if (getdata == NULL) { + PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); + return -1; + } + + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->getdata == getdata); + matched->refcount += 1; + goto finally; + } + + res = _xidregistry_add_type(xidregistry, cls, getdata); + +finally: + PyThread_release_lock(xidregistry->mutex); + return res; +} + +int +_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +{ + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->refcount > 0); + matched->refcount -= 1; + if (matched->refcount == 0) { + (void)_xidregistry_remove_entry(xidregistry, matched); + } + res = 1; + } + + PyThread_release_lock(xidregistry->mutex); + return res; +} + + +/* Cross-interpreter objects are looked up by exact match on the class. + We can reassess this policy when we move from a global registry to a + tp_* slot. */ + +crossinterpdatafunc +_PyCrossInterpreterData_Lookup(PyObject *obj) +{ + PyTypeObject *cls = Py_TYPE(obj); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + + PyThread_release_lock(xidregistry->mutex); + return func; +} + +/* cross-interpreter data for builtin types */ + +struct _shared_bytes_data { + char *bytes; + Py_ssize_t len; +}; + +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); + return PyBytes_FromStringAndSize(shared->bytes, shared->len); +} + +static int +_bytes_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_bytes_data), obj, + _new_bytes_object + ) < 0) + { + return -1; + } + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; + if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { + _PyCrossInterpreterData_Clear(tstate->interp, data); + return -1; + } + return 0; +} + +struct _shared_str_data { + int kind; + const void *buffer; + Py_ssize_t len; +}; + +static PyObject * +_new_str_object(_PyCrossInterpreterData *data) +{ + struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); + return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); +} + +static int +_str_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_str_data), obj, + _new_str_object + ) < 0) + { + return -1; + } + struct _shared_str_data *shared = (struct _shared_str_data *)data->data; + shared->kind = PyUnicode_KIND(obj); + shared->buffer = PyUnicode_DATA(obj); + shared->len = PyUnicode_GET_LENGTH(obj); + return 0; +} + +static PyObject * +_new_long_object(_PyCrossInterpreterData *data) +{ + return PyLong_FromSsize_t((Py_ssize_t)(data->data)); +} + +static int +_long_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + /* Note that this means the size of shareable ints is bounded by + * sys.maxsize. Hence on 32-bit architectures that is half the + * size of maximum shareable ints on 64-bit. + */ + Py_ssize_t value = PyLong_AsSsize_t(obj); + if (value == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); + } + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, + _new_long_object); + // data->obj and data->free remain NULL + return 0; +} + +static PyObject * +_new_none_object(_PyCrossInterpreterData *data) +{ + // XXX Singleton refcounts are problematic across interpreters... + return Py_NewRef(Py_None); +} + +static int +_none_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, + _new_none_object); + // data->data, data->obj and data->free remain NULL + return 0; +} + +static void +_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) +{ + // None + if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + Py_FatalError("could not register None for cross-interpreter sharing"); + } + + // int + if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { + Py_FatalError("could not register int for cross-interpreter sharing"); + } + + // bytes + if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { + Py_FatalError("could not register bytes for cross-interpreter sharing"); + } + + // str + if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + Py_FatalError("could not register str for cross-interpreter sharing"); + } +} diff --git a/Python/pystate.c b/Python/pystate.c index c44a28ca6d3ac..d97a03caf491c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -16,7 +16,6 @@ #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() /* -------------------------------------------------------------------------- CAUTION @@ -495,7 +494,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) return _PyStatus_OK(); } -static void _xidregistry_clear(struct _xidregistry *); +// This is defined in crossinterp.c (for now). +extern void _Py_xidregistry_clear(struct _xidregistry *); void _PyRuntimeState_Fini(_PyRuntimeState *runtime) @@ -505,7 +505,7 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) assert(runtime->object_state.interpreter_leaks == 0); #endif - _xidregistry_clear(&runtime->xidregistry); + _Py_xidregistry_clear(&runtime->xidregistry); if (gilstate_tss_initialized(runtime)) { gilstate_tss_fini(runtime); @@ -948,7 +948,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); - _xidregistry_clear(&interp->xidregistry); + _Py_xidregistry_clear(&interp->xidregistry); /* The lock is owned by the runtime, so we don't free it here. */ interp->xidregistry.mutex = NULL; @@ -2415,593 +2415,6 @@ PyGILState_Release(PyGILState_STATE oldstate) } -/**************************/ -/* cross-interpreter data */ -/**************************/ - -/* cross-interpreter data */ - -static inline void -_xidata_init(_PyCrossInterpreterData *data) -{ - // If the value is being reused - // then _xidata_clear() should have been called already. - assert(data->data == NULL); - assert(data->obj == NULL); - *data = (_PyCrossInterpreterData){0}; - data->interpid = -1; -} - -static inline void -_xidata_clear(_PyCrossInterpreterData *data) -{ - // _PyCrossInterpreterData only has two members that need to be - // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. - // In both cases the original (owning) interpreter must be used, - // which is the caller's responsibility to ensure. - if (data->data != NULL) { - if (data->free != NULL) { - data->free(data->data); - } - data->data = NULL; - } - Py_CLEAR(data->obj); -} - -void -_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - void *shared, PyObject *obj, - xid_newobjectfunc new_object) -{ - assert(data != NULL); - assert(new_object != NULL); - _xidata_init(data); - data->data = shared; - if (obj != NULL) { - assert(interp != NULL); - // released in _PyCrossInterpreterData_Clear() - data->obj = Py_NewRef(obj); - } - // Ideally every object would know its owning interpreter. - // Until then, we have to rely on the caller to identify it - // (but we don't need it in all cases). - data->interpid = (interp != NULL) ? interp->id : -1; - data->new_object = new_object; -} - -int -_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - const size_t size, PyObject *obj, - xid_newobjectfunc new_object) -{ - assert(size > 0); - // For now we always free the shared data in the same interpreter - // where it was allocated, so the interpreter is required. - assert(interp != NULL); - _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); - data->data = PyMem_RawMalloc(size); - if (data->data == NULL) { - return -1; - } - data->free = PyMem_RawFree; - return 0; -} - -void -_PyCrossInterpreterData_Clear(PyInterpreterState *interp, - _PyCrossInterpreterData *data) -{ - assert(data != NULL); - // This must be called in the owning interpreter. - assert(interp == NULL || data->interpid == interp->id); - _xidata_clear(data); -} - -static int -_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) -{ - // data->data can be anything, including NULL, so we don't check it. - - // data->obj may be NULL, so we don't check it. - - if (data->interpid < 0) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing interp"); - return -1; - } - - if (data->new_object == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing new_object func"); - return -1; - } - - // data->free may be NULL, so we don't check it. - - return 0; -} - -crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); - -/* This is a separate func from _PyCrossInterpreterData_Lookup in order - to keep the registry code separate. */ -static crossinterpdatafunc -_lookup_getdata(PyObject *obj) -{ - crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); - if (getdata == NULL && PyErr_Occurred() == 0) - PyErr_Format(PyExc_ValueError, - "%S does not support cross-interpreter data", obj); - return getdata; -} - -int -_PyObject_CheckCrossInterpreterData(PyObject *obj) -{ - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { - return -1; - } - return 0; -} - -int -_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) -{ - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); -#ifdef Py_DEBUG - // The caller must hold the GIL - _Py_EnsureTstateNotNULL(tstate); -#endif - PyInterpreterState *interp = tstate->interp; - - // Reset data before re-populating. - *data = (_PyCrossInterpreterData){0}; - data->interpid = -1; - - // Call the "getdata" func for the object. - Py_INCREF(obj); - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { - Py_DECREF(obj); - return -1; - } - int res = getdata(tstate, obj, data); - Py_DECREF(obj); - if (res != 0) { - return -1; - } - - // Fill in the blanks and validate the result. - data->interpid = interp->id; - if (_check_xidata(tstate, data) != 0) { - (void)_PyCrossInterpreterData_Release(data); - return -1; - } - - return 0; -} - -PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) -{ - return data->new_object(data); -} - -int -_Py_CallInInterpreter(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == current_fast_get(interp->runtime)->interp) { - return func(arg); - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); - return 0; -} - -int -_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == current_fast_get(interp->runtime)->interp) { - int res = func(arg); - PyMem_RawFree(arg); - return res; - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); - return 0; -} - -static int -_call_clear_xidata(void *data) -{ - _xidata_clear((_PyCrossInterpreterData *)data); - return 0; -} - -static int -_xidata_release(_PyCrossInterpreterData *data, int rawfree) -{ - if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { - // Nothing to release! - if (rawfree) { - PyMem_RawFree(data); - } - else { - data->data = NULL; - } - return 0; - } - - // Switch to the original interpreter. - PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interpid); - if (interp == NULL) { - // The interpreter was already destroyed. - // This function shouldn't have been called. - // XXX Someone leaked some memory... - assert(PyErr_Occurred()); - if (rawfree) { - PyMem_RawFree(data); - } - return -1; - } - - // "Release" the data and/or the object. - if (rawfree) { - return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, data); - } - else { - return _Py_CallInInterpreter(interp, _call_clear_xidata, data); - } -} - -int -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) -{ - return _xidata_release(data, 0); -} - -int -_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) -{ - return _xidata_release(data, 1); -} - -/* registry of {type -> crossinterpdatafunc} */ - -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -static int -_xidregistry_add_type(struct _xidregistry *xidregistry, - PyTypeObject *cls, crossinterpdatafunc getdata) -{ - struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); - if (newhead == NULL) { - return -1; - } - *newhead = (struct _xidregitem){ - // We do not keep a reference, to avoid keeping the class alive. - .cls = cls, - .refcount = 1, - .getdata = getdata, - }; - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - // XXX Assign a callback to clear the entry from the registry? - newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); - if (newhead->weakref == NULL) { - PyMem_RawFree(newhead); - return -1; - } - } - newhead->next = xidregistry->head; - if (newhead->next != NULL) { - newhead->next->prev = newhead; - } - xidregistry->head = newhead; - return 0; -} - -static struct _xidregitem * -_xidregistry_remove_entry(struct _xidregistry *xidregistry, - struct _xidregitem *entry) -{ - struct _xidregitem *next = entry->next; - if (entry->prev != NULL) { - assert(entry->prev->next == entry); - entry->prev->next = next; - } - else { - assert(xidregistry->head == entry); - xidregistry->head = next; - } - if (next != NULL) { - next->prev = entry->prev; - } - Py_XDECREF(entry->weakref); - PyMem_RawFree(entry); - return next; -} - -static void -_xidregistry_clear(struct _xidregistry *xidregistry) -{ - struct _xidregitem *cur = xidregistry->head; - xidregistry->head = NULL; - while (cur != NULL) { - struct _xidregitem *next = cur->next; - Py_XDECREF(cur->weakref); - PyMem_RawFree(cur); - cur = next; - } -} - -static struct _xidregitem * -_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) -{ - struct _xidregitem *cur = xidregistry->head; - while (cur != NULL) { - if (cur->weakref != NULL) { - // cur is/was a heap type. - PyObject *registered = _PyWeakref_GET_REF(cur->weakref); - if (registered == NULL) { - // The weakly ref'ed object was freed. - cur = _xidregistry_remove_entry(xidregistry, cur); - continue; - } - assert(PyType_Check(registered)); - assert(cur->cls == (PyTypeObject *)registered); - assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); - Py_DECREF(registered); - } - if (cur->cls == cls) { - return cur; - } - cur = cur->next; - } - return NULL; -} - -static inline struct _xidregistry * -_get_xidregistry(PyInterpreterState *interp, PyTypeObject *cls) -{ - struct _xidregistry *xidregistry = &interp->runtime->xidregistry; - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - assert(interp->xidregistry.mutex == xidregistry->mutex); - xidregistry = &interp->xidregistry; - } - return xidregistry; -} - -static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); - -static inline void -_ensure_builtins_xid(PyInterpreterState *interp, struct _xidregistry *xidregistry) -{ - if (xidregistry != &interp->xidregistry) { - assert(xidregistry == &interp->runtime->xidregistry); - if (xidregistry->head == NULL) { - _register_builtins_for_crossinterpreter_data(xidregistry); - } - } -} - -int -_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) -{ - if (!PyType_Check(cls)) { - PyErr_Format(PyExc_ValueError, "only classes may be registered"); - return -1; - } - if (getdata == NULL) { - PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); - return -1; - } - - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - _ensure_builtins_xid(interp, xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->getdata == getdata); - matched->refcount += 1; - goto finally; - } - - res = _xidregistry_add_type(xidregistry, cls, getdata); - -finally: - PyThread_release_lock(xidregistry->mutex); - return res; -} - -int -_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) -{ - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->refcount > 0); - matched->refcount -= 1; - if (matched->refcount == 0) { - (void)_xidregistry_remove_entry(xidregistry, matched); - } - res = 1; - } - - PyThread_release_lock(xidregistry->mutex); - return res; -} - - -/* Cross-interpreter objects are looked up by exact match on the class. - We can reassess this policy when we move from a global registry to a - tp_* slot. */ - -crossinterpdatafunc -_PyCrossInterpreterData_Lookup(PyObject *obj) -{ - PyTypeObject *cls = Py_TYPE(obj); - - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - _ensure_builtins_xid(interp, xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; - - PyThread_release_lock(xidregistry->mutex); - return func; -} - -/* cross-interpreter data for builtin types */ - -struct _shared_bytes_data { - char *bytes; - Py_ssize_t len; -}; - -static PyObject * -_new_bytes_object(_PyCrossInterpreterData *data) -{ - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); - return PyBytes_FromStringAndSize(shared->bytes, shared->len); -} - -static int -_bytes_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_bytes_data), obj, - _new_bytes_object - ) < 0) - { - return -1; - } - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; - if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { - _PyCrossInterpreterData_Clear(tstate->interp, data); - return -1; - } - return 0; -} - -struct _shared_str_data { - int kind; - const void *buffer; - Py_ssize_t len; -}; - -static PyObject * -_new_str_object(_PyCrossInterpreterData *data) -{ - struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); - return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); -} - -static int -_str_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_str_data), obj, - _new_str_object - ) < 0) - { - return -1; - } - struct _shared_str_data *shared = (struct _shared_str_data *)data->data; - shared->kind = PyUnicode_KIND(obj); - shared->buffer = PyUnicode_DATA(obj); - shared->len = PyUnicode_GET_LENGTH(obj); - return 0; -} - -static PyObject * -_new_long_object(_PyCrossInterpreterData *data) -{ - return PyLong_FromSsize_t((Py_ssize_t)(data->data)); -} - -static int -_long_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - /* Note that this means the size of shareable ints is bounded by - * sys.maxsize. Hence on 32-bit architectures that is half the - * size of maximum shareable ints on 64-bit. - */ - Py_ssize_t value = PyLong_AsSsize_t(obj); - if (value == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); - } - return -1; - } - _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, - _new_long_object); - // data->obj and data->free remain NULL - return 0; -} - -static PyObject * -_new_none_object(_PyCrossInterpreterData *data) -{ - // XXX Singleton refcounts are problematic across interpreters... - return Py_NewRef(Py_None); -} - -static int -_none_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, - _new_none_object); - // data->data, data->obj and data->free remain NULL - return 0; -} - -static void -_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) -{ - // None - if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { - Py_FatalError("could not register None for cross-interpreter sharing"); - } - - // int - if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { - Py_FatalError("could not register int for cross-interpreter sharing"); - } - - // bytes - if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { - Py_FatalError("could not register bytes for cross-interpreter sharing"); - } - - // str - if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { - Py_FatalError("could not register str for cross-interpreter sharing"); - } -} - - /*************/ /* Other API */ /*************/

1 0

gh-110697: Use CLOCK_RES in test_os.TimerfdTests (#111529)
by vstinner Oct. 30, 2023

Oct. 30, 2023

https://github.com/python/cpython/commit/7b153d14ef2e446986148ce7833a00bee2… commit: 7b153d14ef2e446986148ce7833a00bee26208f1 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-10-30T22:47:59Z summary: gh-110697: Use CLOCK_RES in test_os.TimerfdTests (#111529) More TimerfdTests tests tolerate a difference of 1 ms in measured elapsed time. files: M Lib/test/test_os.py diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c23de4dbe2b63..22e40d9557be7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4029,7 +4029,7 @@ def test_timerfd_interval(self): t = time.perf_counter() - t total_time = initial_expiration + interval * (count - 1) - self.assertGreater(t, total_time) + self.assertGreater(t, total_time - self.CLOCK_RES) # wait 3.5 time of interval time.sleep( (count+0.5) * interval) @@ -4083,7 +4083,7 @@ def test_timerfd_select(self): t = time.perf_counter() - t total_time = initial_expiration + interval * (count - 1) - self.assertGreater(t, total_time) + self.assertGreater(t, total_time - self.CLOCK_RES) def check_timerfd_poll(self, nanoseconds): fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) @@ -4129,10 +4129,10 @@ def check_timerfd_poll(self, nanoseconds): total_time = initial_expiration_ns + interval_ns * (count - 1) if nanoseconds: dt = time.perf_counter_ns() - t - self.assertGreater(dt, total_time) + self.assertGreater(dt, total_time - self.CLOCK_RES_NS) else: dt = time.perf_counter() - t - self.assertGreater(dt, total_time / sec_to_nsec) + self.assertGreater(dt, total_time / sec_to_nsec - self.CLOCK_RES) selector.unregister(fd) def test_timerfd_poll(self): @@ -4189,7 +4189,7 @@ def test_timerfd_ns_interval(self): t = time.perf_counter_ns() - t total_time_ns = initial_expiration_ns + interval_ns * (count - 1) - self.assertGreater(t, total_time_ns) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) # wait 3.5 time of interval time.sleep( (count+0.5) * interval_ns / one_sec_in_nsec) @@ -4248,7 +4248,7 @@ def test_timerfd_ns_select(self): t = time.perf_counter_ns() - t total_time_ns = initial_expiration_ns + interval_ns * (count - 1) - self.assertGreater(t, total_time_ns) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) class OSErrorTests(unittest.TestCase): def setUp(self):

1 0