Mailman 3 April 2024 - Python-checkins

gh-118201 - Disable the flaky POSIX test_confstr test on iOS (GH-118452)
by ned-deily April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/9c468e2c5dffb6fa9811fd16e70fa0463b… commit: 9c468e2c5dffb6fa9811fd16e70fa0463bdfce5f branch: main author: Russell Keith-Magee <russell(a)keith-magee.com> committer: ned-deily <nad(a)python.org> date: 2024-04-30T19:32:37-04:00 summary: gh-118201 - Disable the flaky POSIX test_confstr test on iOS (GH-118452) files: M Lib/test/test_posix.py diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 1d22869046fd12..7e5f04c22bd6d3 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -564,6 +564,7 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_confstr(self): self.assertRaises(ValueError, posix.confstr, "CS_garbage") self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True)

1 0

gh-117657: Don't specialize RESUME_CHECK when specialization is disabled (GH-118349)
by Fidget-Spinner April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/7fabcc727dee52a3e0dfe4f903ad414e93… commit: 7fabcc727dee52a3e0dfe4f903ad414e93cf2dc9 branch: main author: Ken Jin <kenjin(a)python.org> committer: Fidget-Spinner <kenjin4096(a)gmail.com> date: 2024-04-30T21:51:59Z summary: gh-117657: Don't specialize RESUME_CHECK when specialization is disabled (GH-118349) files: M Python/bytecodes.c M Python/generated_cases.c.h diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5bb7e1211385a5..18837aef74d78e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -163,7 +163,9 @@ dummy_func( if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } + #if ENABLE_SPECIALIZATION FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a5bb29385844d8..1444f5cdebba4b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4955,7 +4955,9 @@ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } + #if ENABLE_SPECIALIZATION FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ } DISPATCH(); }

1 0

gh-117139: Add header for tagged pointers (GH-118330)
by Fidget-Spinner April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/dc6b12d1b2ea26bb0323d932fa7b1a337e… commit: dc6b12d1b2ea26bb0323d932fa7b1a337eaca562 branch: main author: Ken Jin <kenjin(a)python.org> committer: Fidget-Spinner <kenjin4096(a)gmail.com> date: 2024-05-01T04:46:13+08:00 summary: gh-117139: Add header for tagged pointers (GH-118330) --------- Co-authored-by: Sam Gross <655866+colesbury(a)users.noreply.github.com> files: A Include/internal/pycore_stackref.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h new file mode 100644 index 00000000000000..fd929cd4873a8b --- /dev/null +++ b/Include/internal/pycore_stackref.h @@ -0,0 +1,195 @@ +#ifndef Py_INTERNAL_STACKREF_H +#define Py_INTERNAL_STACKREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include <stddef.h> + +typedef union { + uintptr_t bits; +} _PyStackRef; + +static const _PyStackRef Py_STACKREF_NULL = { .bits = 0 }; + +#define Py_TAG_DEFERRED (1) + +// Gets a PyObject * from a _PyStackRef +#if defined(Py_GIL_DISABLED) +static inline PyObject * +PyStackRef_Get(_PyStackRef tagged) +{ + PyObject *cleared = ((PyObject *)((tagged).bits & (~Py_TAG_DEFERRED))); + return cleared; +} +#else +# define PyStackRef_Get(tagged) ((PyObject *)((tagged).bits)) +#endif + +// Converts a PyObject * to a PyStackRef, stealing the reference. +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_StealRef(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + return ((_PyStackRef){.bits = ((uintptr_t)(obj))}); +} +# define PyStackRef_StealRef(obj) _PyStackRef_StealRef(_PyObject_CAST(obj)) +#else +# define PyStackRef_StealRef(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) +#endif + +// Converts a PyObject * to a PyStackRef, with a new reference +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_NewRefDeferred(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + assert(obj != NULL); + if (_PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + } + else { + return (_PyStackRef){ .bits = (uintptr_t)Py_NewRef(obj) }; + } +} +# define PyStackRef_NewRefDeferred(obj) _PyStackRef_NewRefDeferred(_PyObject_CAST(obj)) +#else +# define PyStackRef_NewRefDeferred(obj) PyStackRef_NewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) +#endif + +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_XNewRefDeferred(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + if (obj == NULL) { + return Py_STACKREF_NULL; + } + return _PyStackRef_NewRefDeferred(obj); +} +# define PyStackRef_XNewRefDeferred(obj) _PyStackRef_XNewRefDeferred(_PyObject_CAST(obj)) +#else +# define PyStackRef_XNewRefDeferred(obj) PyStackRef_XNewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) +#endif + +// Converts a PyStackRef back to a PyObject *. +#if defined(Py_GIL_DISABLED) +static inline PyObject * +PyStackRef_StealObject(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); + return Py_NewRef(PyStackRef_Get(tagged)); + } + return PyStackRef_Get(tagged); +} +#else +# define PyStackRef_StealObject(tagged) PyStackRef_Get(tagged) +#endif + +static inline void +_Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length) +{ + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_Get(src[i]); + } +} + +static inline void +_Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length) +{ + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_StealObject(src[i]); + } +} + + +#define PyStackRef_XSETREF(dst, src) \ + do { \ + _PyStackRef *_tmp_dst_ptr = &(dst) \ + _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + PyStackRef_XDECREF(_tmp_old_dst); \ + } while (0) + +#define PyStackRef_SETREF(dst, src) \ + do { \ + _PyStackRef *_tmp_dst_ptr = &(dst); \ + _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + PyStackRef_DECREF(_tmp_old_dst); \ + } while (0) + +#define PyStackRef_CLEAR(op) \ + do { \ + _PyStackRef *_tmp_op_ptr = &(op); \ + _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op.bits != Py_STACKREF_NULL.bits) { \ + *_tmp_op_ptr = Py_STACKREF_NULL; \ + PyStackRef_DECREF(_tmp_old_op); \ + } \ + } while (0) + +#if defined(Py_GIL_DISABLED) +static inline void +PyStackRef_DECREF(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + return; + } + Py_DECREF(PyStackRef_Get(tagged)); +} +#else +# define PyStackRef_DECREF(op) Py_DECREF(PyStackRef_Get(op)) +#endif + +#if defined(Py_GIL_DISABLED) +static inline void +PyStackRef_INCREF(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); + return; + } + Py_INCREF(PyStackRef_Get(tagged)); +} +#else +# define PyStackRef_INCREF(op) Py_INCREF(PyStackRef_Get(op)) +#endif + +static inline void +PyStackRef_XDECREF(_PyStackRef op) +{ + if (op.bits != Py_STACKREF_NULL.bits) { + PyStackRef_DECREF(op); + } +} + +static inline _PyStackRef +PyStackRef_NewRef(_PyStackRef obj) +{ + PyStackRef_INCREF(obj); + return obj; +} + +static inline _PyStackRef +PyStackRef_XNewRef(_PyStackRef obj) +{ + if (obj.bits == Py_STACKREF_NULL.bits) { + return obj; + } + return PyStackRef_NewRef(obj); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STACKREF_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 4e4d01d14ee3cb..e69d1fe6e2dd14 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1225,6 +1225,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_stackref.h \ $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 25d52945c1c330..4cb3e0d3227092 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -291,6 +291,7 @@ <ClInclude Include="..\Include\internal\pycore_structseq.h" /> <ClInclude Include="..\Include\internal\pycore_sysmodule.h" /> <ClInclude Include="..\Include\internal\pycore_symtable.h" /> + <ClInclude Include="..\Include\internal\pycore_stackref.h" /> <ClInclude Include="..\Include\internal\pycore_time.h" /> <ClInclude Include="..\Include\internal\pycore_token.h" /> <ClInclude Include="..\Include\internal\pycore_traceback.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4b1f9aa6538562..0b858cf1eb46a8 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -789,6 +789,9 @@ <ClInclude Include="..\Include\internal\pycore_symtable.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_stackref.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_time.h"> <Filter>Include\internal</Filter> </ClInclude>

1 0

gh-118422: Fix run_fileexflags() test (#118429)
by vstinner April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/e93c39b47ea623dfaf61f80775ad4747b1… commit: e93c39b47ea623dfaf61f80775ad4747b163efe5 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2024-04-30T22:32:55+02:00 summary: gh-118422: Fix run_fileexflags() test (#118429) Don't test the undefined behavior of fileno() on a closed file, but use fstat() as a reliable test if the file was closed or not. files: M Include/internal/pycore_fileutils.h M Modules/_testcapi/run.c M Python/fileutils.c M Python/pylifecycle.c diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index bc8100b58e8ea3..13f86b01bbfe8f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -326,6 +326,9 @@ extern int _PyFile_Flush(PyObject *); extern int _Py_GetTicksPerSecond(long *ticks_per_second); #endif +// Export for '_testcapi' shared extension +PyAPI_FUNC(int) _Py_IsValidFD(int fd); + #ifdef __cplusplus } #endif diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c index 4fd98b82d762ff..21244d02967ebf 100644 --- a/Modules/_testcapi/run.c +++ b/Modules/_testcapi/run.c @@ -1,5 +1,7 @@ +#define PYTESTCAPI_NEED_INTERNAL_API #include "parts.h" #include "util.h" +#include "pycore_fileutils.h" // _Py_IsValidFD() #include <stdio.h> #include <errno.h> @@ -71,21 +73,18 @@ run_fileexflags(PyObject *mod, PyObject *pos_args) PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); return NULL; } + int fd = fileno(fp); result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); -#if defined(__linux__) || defined(MS_WINDOWS) || defined(__APPLE__) - /* The behavior of fileno() after fclose() is undefined, but it is - * the only practical way to check whether the file was closed. - * Only test this on the known platforms. */ - if (closeit && result && fileno(fp) >= 0) { + if (closeit && result && _Py_IsValidFD(fd)) { PyErr_SetString(PyExc_AssertionError, "File was not closed after excution"); Py_DECREF(result); fclose(fp); return NULL; } -#endif - if (!closeit && fileno(fp) < 0) { + + if (!closeit && !_Py_IsValidFD(fd)) { PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after excution"); Py_XDECREF(result); return NULL; diff --git a/Python/fileutils.c b/Python/fileutils.c index 54853ba2f75d9d..e6a5391a3a28b5 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -3050,3 +3050,52 @@ _Py_GetTicksPerSecond(long *ticks_per_second) return 0; } #endif + + +/* Check if a file descriptor is valid or not. + Return 0 if the file descriptor is invalid, return non-zero otherwise. */ +int +_Py_IsValidFD(int fd) +{ +/* dup() is faster than fstat(): fstat() can require input/output operations, + whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python + startup. Problem: dup() doesn't check if the file descriptor is valid on + some platforms. + + fcntl(fd, F_GETFD) is even faster, because it only checks the process table. + It is preferred over dup() when available, since it cannot fail with the + "too many open files" error (EMFILE). + + bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other + side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with + EBADF. FreeBSD has similar issue (bpo-32849). + + Only use dup() on Linux where dup() is enough to detect invalid FD + (bpo-32849). +*/ + if (fd < 0) { + return 0; + } +#if defined(F_GETFD) && ( \ + defined(__linux__) || \ + defined(__APPLE__) || \ + (defined(__wasm__) && !defined(__wasi__))) + return fcntl(fd, F_GETFD) >= 0; +#elif defined(__linux__) + int fd2 = dup(fd); + if (fd2 >= 0) { + close(fd2); + } + return (fd2 >= 0); +#elif defined(MS_WINDOWS) + HANDLE hfile; + _Py_BEGIN_SUPPRESS_IPH + hfile = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + return (hfile != INVALID_HANDLE_VALUE + && GetFileType(hfile) != FILE_TYPE_UNKNOWN); +#else + struct stat st; + return (fstat(fd, &st) == 0); +#endif +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a672d8c7915ccd..790398e944f58e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2410,54 +2410,6 @@ init_import_site(void) return _PyStatus_OK(); } -/* Check if a file descriptor is valid or not. - Return 0 if the file descriptor is invalid, return non-zero otherwise. */ -static int -is_valid_fd(int fd) -{ -/* dup() is faster than fstat(): fstat() can require input/output operations, - whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python - startup. Problem: dup() doesn't check if the file descriptor is valid on - some platforms. - - fcntl(fd, F_GETFD) is even faster, because it only checks the process table. - It is preferred over dup() when available, since it cannot fail with the - "too many open files" error (EMFILE). - - bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other - side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with - EBADF. FreeBSD has similar issue (bpo-32849). - - Only use dup() on Linux where dup() is enough to detect invalid FD - (bpo-32849). -*/ - if (fd < 0) { - return 0; - } -#if defined(F_GETFD) && ( \ - defined(__linux__) || \ - defined(__APPLE__) || \ - defined(__wasm__)) - return fcntl(fd, F_GETFD) >= 0; -#elif defined(__linux__) - int fd2 = dup(fd); - if (fd2 >= 0) { - close(fd2); - } - return (fd2 >= 0); -#elif defined(MS_WINDOWS) - HANDLE hfile; - _Py_BEGIN_SUPPRESS_IPH - hfile = (HANDLE)_get_osfhandle(fd); - _Py_END_SUPPRESS_IPH - return (hfile != INVALID_HANDLE_VALUE - && GetFileType(hfile) != FILE_TYPE_UNKNOWN); -#else - struct stat st; - return (fstat(fd, &st) == 0); -#endif -} - /* returns Py_None if the fd is not valid */ static PyObject* create_stdio(const PyConfig *config, PyObject* io, @@ -2471,8 +2423,9 @@ create_stdio(const PyConfig *config, PyObject* io, int buffering, isatty; const int buffered_stdio = config->buffered_stdio; - if (!is_valid_fd(fd)) + if (!_Py_IsValidFD(fd)) { Py_RETURN_NONE; + } /* stdin is always opened in buffered mode, first because it shouldn't make a difference in common use cases, second because TextIOWrapper @@ -2588,9 +2541,9 @@ create_stdio(const PyConfig *config, PyObject* io, Py_XDECREF(text); Py_XDECREF(raw); - if (PyErr_ExceptionMatches(PyExc_OSError) && !is_valid_fd(fd)) { + if (PyErr_ExceptionMatches(PyExc_OSError) && !_Py_IsValidFD(fd)) { /* Issue #24891: the file descriptor was closed after the first - is_valid_fd() check was called. Ignore the OSError and set the + _Py_IsValidFD() check was called. Ignore the OSError and set the stream to None. */ PyErr_Clear(); Py_RETURN_NONE;

1 0

gh-118124: Use static_assert() in Py_BUILD_ASSERT() on C11 (#118398)
by vstinner April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/587388ff22dc7cfa4b66722daf0d33cd80… commit: 587388ff22dc7cfa4b66722daf0d33cd804af9f2 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2024-04-30T22:29:48+02:00 summary: gh-118124: Use static_assert() in Py_BUILD_ASSERT() on C11 (#118398) Use static_assert() in Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() on C11 and newer and C++11 and newer. Add tests to test_cext and test_cppext. files: A Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst M Include/pymacro.h M Lib/test/test_cext/extension.c M Lib/test/test_cppext/extension.cpp diff --git a/Include/pymacro.h b/Include/pymacro.h index cd6fc4eba9c2ed..b388c2a4a663ce 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -46,24 +46,41 @@ /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) -/* Assert a build-time dependency, as an expression. - - Your compile will fail if the condition isn't true, or can't be evaluated - by the compiler. This can be used in an expression: its value is 0. - - Example: - - #define foo_to_char(foo) \ - ((char *)(foo) \ - + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) - - Written by Rusty Russell, public domain, http://ccodearchive.net/ */ -#define Py_BUILD_ASSERT_EXPR(cond) \ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define Py_BUILD_ASSERT_EXPR(cond) \ + ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \ + 0) +#else + /* Assert a build-time dependency, as an expression. + * + * Your compile will fail if the condition isn't true, or can't be evaluated + * by the compiler. This can be used in an expression: its value is 0. + * + * Example: + * + * #define foo_to_char(foo) \ + * ((char *)(foo) \ + * + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + * + * Written by Rusty Russell, public domain, http://ccodearchive.net/ + */ +# define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#endif -#define Py_BUILD_ASSERT(cond) do { \ - (void)Py_BUILD_ASSERT_EXPR(cond); \ - } while(0) +#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L)) + // Use static_assert() on C11 and newer +# define Py_BUILD_ASSERT(cond) \ + do { \ + static_assert((cond), #cond); \ + } while (0) +#else +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) +#endif /* Get the number of elements in a visible array diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 662abd4005e8d2..eb23dbe20353ba 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -44,6 +44,11 @@ _testcext_exec(PyObject *module) return -1; } #endif + + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index a569c2251d1ad7..ab485b629b7788 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -225,6 +225,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } diff --git a/Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst b/Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst new file mode 100644 index 00000000000000..3deeb517eb9fce --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst @@ -0,0 +1,3 @@ +Fix :c:macro:`Py_BUILD_ASSERT` and :c:macro:`Py_BUILD_ASSERT_EXPR` for +non-constant expressions: use ``static_assert()`` on C11 and newer. +Patch by Victor Stinner.

1 0

gh-118218: Reuse return tuple in itertools.pairwise (GH-118219)
by serhiy-storchaka April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/6999d68d2878871493d85dc63599f3d44e… commit: 6999d68d2878871493d85dc63599f3d44eada104 branch: main author: Shantanu <12621235+hauntsaninja(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-04-30T23:16:52+03:00 summary: gh-118218: Reuse return tuple in itertools.pairwise (GH-118219) files: A Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst M Lib/test/test_itertools.py M Modules/itertoolsmodule.c diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 95e67911db6a7f..e243da309f03d8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1821,6 +1821,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject diff --git a/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst new file mode 100644 index 00000000000000..6d3c54cb485655 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst @@ -0,0 +1 @@ +Speed up :func:`itertools.pairwise` in the common case by up to 1.8x. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6ee447ef6a8cd6..21ce3ecfad0354 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -270,6 +270,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; + PyObject *result; } pairwiseobject; /*[clinic input] @@ -301,6 +302,11 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; + po->result = PyTuple_Pack(2, Py_None, Py_None); + if (po->result == NULL) { + Py_DECREF(po); + return NULL; + } return (PyObject *)po; } @@ -311,6 +317,7 @@ pairwise_dealloc(pairwiseobject *po) PyObject_GC_UnTrack(po); Py_XDECREF(po->it); Py_XDECREF(po->old); + Py_XDECREF(po->result); tp->tp_free(po); Py_DECREF(tp); } @@ -321,6 +328,7 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) Py_VISIT(Py_TYPE(po)); Py_VISIT(po->it); Py_VISIT(po->old); + Py_VISIT(po->result); return 0; } @@ -355,8 +363,30 @@ pairwise_next(pairwiseobject *po) Py_DECREF(old); return NULL; } - /* Future optimization: Reuse the result tuple as we do in enumerate() */ - result = PyTuple_Pack(2, old, new); + + result = po->result; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + PyObject *last_old = PyTuple_GET_ITEM(result, 0); + PyObject *last_new = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + Py_DECREF(last_old); + Py_DECREF(last_new); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result != NULL) { + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + } + } + Py_XSETREF(po->old, new); Py_DECREF(old); return result;

1 0

gh-118406: Add signature for sqlite3.Connection objects (#118428)
by erlend-aasland April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/b568c2c1ff5c0b1922a6402dc95c588d7f… commit: b568c2c1ff5c0b1922a6402dc95c588d7f9aa914 branch: main author: Erlend E. Aasland <erlend(a)python.org> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2024-04-30T19:58:22Z summary: gh-118406: Add signature for sqlite3.Connection objects (#118428) files: A Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst M Lib/test/test_sqlite3/test_dbapi.py M Modules/_sqlite/connection.c diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 6d8744ca5f7969..51ce095df41fc1 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -590,6 +590,11 @@ def test_connection_resource_warning(self): del cx gc_collect() + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst b/Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst new file mode 100644 index 00000000000000..c60ddf9e00498e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst @@ -0,0 +1 @@ +Add signature for :class:`sqlite3.Connection` objects. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 74984ca5365743..fc03e4a085c179 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -2561,6 +2561,12 @@ set_autocommit(pysqlite_Connection *self, PyObject *val, void *Py_UNUSED(ctx)) return 0; } +static PyObject * +get_sig(PyObject *self, void *Py_UNUSED(ctx)) +{ + return PyUnicode_FromString("(sql, /)"); +} + static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); @@ -2570,6 +2576,7 @@ static PyGetSetDef connection_getset[] = { {"total_changes", (getter)pysqlite_connection_get_total_changes, (setter)0}, {"in_transaction", (getter)pysqlite_connection_get_in_transaction, (setter)0}, {"autocommit", (getter)get_autocommit, (setter)set_autocommit}, + {"__text_signature__", get_sig, (setter)0}, {NULL} };

1 0

[gh-117657] Fix some issues with TSAN in typeobject (#118249)
by DinoV April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/19d468a3ab1d57c07a4283d67d43990857… commit: 19d468a3ab1d57c07a4283d67d439908574aa0cc branch: main author: Dino Viehland <dinoviehland(a)meta.com> committer: DinoV <dinoviehland(a)gmail.com> date: 2024-04-30T12:37:38-07:00 summary: [gh-117657] Fix some issues with TSAN in typeobject (#118249) Fix some racing reads in typebobject.c files: M Include/internal/pycore_pyatomic_ft_wrappers.h M Objects/typeobject.c diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 83f02c92495b3f..bc6aba56cf9fc7 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -43,6 +43,8 @@ extern "C" { _Py_atomic_load_uint8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ _Py_atomic_load_uint16_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ + _Py_atomic_load_uint32_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -55,6 +57,9 @@ extern "C" { _Py_atomic_store_uint8_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ _Py_atomic_store_uint16_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ + _Py_atomic_store_uint32_relaxed(&value, new_value) + #else #define FT_ATOMIC_LOAD_PTR(value) value #define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value @@ -69,12 +74,15 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value + #endif #ifdef __cplusplus diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 50efbb6e182df0..ec19a3d461f623 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -43,7 +43,8 @@ class object "PyObject *" "&PyBaseObject_Type" & ((1 << MCACHE_SIZE_EXP) - 1)) #define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH((type)->tp_version_tag, ((Py_ssize_t)(name)) >> 3) + MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ + ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ PyUnicode_IS_READY(name) && \ @@ -907,7 +908,7 @@ type_modified_unlocked(PyTypeObject *type) } type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -984,7 +985,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1020,7 +1021,8 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, + NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1029,7 +1031,8 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_VERSION_TAG(interp)++; + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, + NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } @@ -5085,7 +5088,9 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) // synchronize-with other writing threads by doing an acquire load on the sequence while (1) { int sequence = _PySeqLock_BeginRead(&entry->sequence); - if (_Py_atomic_load_uint32_relaxed(&entry->version) == type->tp_version_tag && + uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); + uint32_t type_version = _Py_atomic_load_uint32_relaxed(&type->tp_version_tag); + if (entry_version == type_version && _Py_atomic_load_ptr_relaxed(&entry->name) == name) { assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name));

1 0

gh-118332: Fix deadlock involving stop the world (#118412)
by colesbury April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/b2c3b70c7102197e4505e6cd69722dc508… commit: b2c3b70c7102197e4505e6cd69722dc508527d22 branch: main author: Sam Gross <colesbury(a)gmail.com> committer: colesbury <colesbury(a)gmail.com> date: 2024-04-30T15:01:28-04:00 summary: gh-118332: Fix deadlock involving stop the world (#118412) Avoid detaching thread state when stopping the world. When re-attaching the thread state, the thread would attempt to resume the top-most critical section, which might now be held by a thread paused for our stop-the-world request. files: M Include/internal/pycore_lock.h M Modules/_testinternalcapi/test_critical_sections.c M Modules/_threadmodule.c M Python/lock.c M Python/pystate.c diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index f993c95ecbf75a..a5b28e4bd4744e 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -150,8 +150,10 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, -// and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns); +// and 0 if the timeout expired or thread was interrupted. If `detach` is +// true, then the thread will detach/release the GIL while waiting. +PyAPI_FUNC(int) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach); // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index 94da0468fcf149..cdf8a70fc79ff3 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -204,6 +204,90 @@ test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) Py_DECREF(test_data.obj1); Py_RETURN_NONE; } + +static void +pysleep(int ms) +{ +#ifdef MS_WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +struct test_data_gc { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t id; + Py_ssize_t countdown; + PyEvent done_event; + PyEvent ready; +}; + +static void +thread_gc(void *arg) +{ + struct test_data_gc *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + Py_ssize_t id = _Py_atomic_add_ssize(&test_data->id, 1); + if (id == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + else { + // wait for all test threads to more reliably reproduce the issue. + PyEvent_Wait(&test_data->ready); + } + + if (id == 0) { + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + // pause long enough that the lock would be handed off directly to + // a waiting thread. + pysleep(5); + PyGC_Collect(); + Py_END_CRITICAL_SECTION(); + } + else if (id == 1) { + pysleep(1); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + else if (id == 2) { + // sleep long enough so that thread 0 is waiting to stop the world + pysleep(6); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + + PyGILState_Release(gil); + if (_Py_atomic_add_ssize(&test_data->countdown, -1) == 1) { + // last thread to finish sets done_event + _PyEvent_Notify(&test_data->done_event); + } +} + +static PyObject * +test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-118332: Contended critical sections should not deadlock with GC + const Py_ssize_t NUM_THREADS = 3; + struct test_data_gc test_data = { + .obj = PyDict_New(), + .countdown = NUM_THREADS, + .num_threads = NUM_THREADS, + }; + assert(test_data.obj != NULL); + + for (int i = 0; i < NUM_THREADS; i++) { + PyThread_start_new_thread(&thread_gc, &test_data); + } + PyEvent_Wait(&test_data.done_event); + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + #endif static PyMethodDef test_methods[] = { @@ -212,6 +296,7 @@ static PyMethodDef test_methods[] = { {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, + {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, #endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 5aa719c3834e61..f5e3b42600675e 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -501,7 +501,8 @@ ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) // Wait until the deadline for the thread to exit. PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0; - while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns)) { + int detach = 1; + while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns, detach)) { if (deadline) { // _PyDeadline_Get will return a negative value if the deadline has // been exceeded. diff --git a/Python/lock.c b/Python/lock.c index 91c66df8fd9093..5ed95fcaf4188c 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -277,12 +277,12 @@ _PyEvent_Notify(PyEvent *evt) void PyEvent_Wait(PyEvent *evt) { - while (!PyEvent_WaitTimed(evt, -1)) + while (!PyEvent_WaitTimed(evt, -1, /*detach=*/1)) ; } int -PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach) { for (;;) { uint8_t v = _Py_atomic_load_uint8(&evt->v); @@ -298,7 +298,7 @@ PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) uint8_t expected = _Py_HAS_PARKED; (void) _PyParkingLot_Park(&evt->v, &expected, sizeof(evt->v), - timeout_ns, NULL, 1); + timeout_ns, NULL, detach); return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED; } diff --git a/Python/pystate.c b/Python/pystate.c index 78b39c9a577404..9d7b73b3a071e1 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2238,7 +2238,8 @@ stop_the_world(struct _stoptheworld_state *stw) } PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) - if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) { + int detach = 0; + if (PyEvent_WaitTimed(&stw->stop_event, wait_ns, detach)) { assert(stw->thread_countdown == 0); break; }

1 0

gh-117657: Fix small issues with instrumentation and TSAN (#118064)
by DinoV April 30, 2024

April 30, 2024

https://github.com/python/cpython/commit/4a1cf66c5c0afa36d7a51d5f9d3874cda1… commit: 4a1cf66c5c0afa36d7a51d5f9d3874cda10df79c branch: main author: Dino Viehland <dinoviehland(a)meta.com> committer: DinoV <dinoviehland(a)gmail.com> date: 2024-04-30T11:38:05-07:00 summary: gh-117657: Fix small issues with instrumentation and TSAN (#118064) Small TSAN fixups for instrumentation files: M Include/internal/pycore_pyatomic_ft_wrappers.h M Objects/genobject.c M Python/bytecodes.c M Python/ceval.c M Python/ceval_macros.h M Python/generated_cases.c.h M Python/instrumentation.c diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index bbfc462a733d0e..83f02c92495b3f 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -39,6 +39,10 @@ extern "C" { _Py_atomic_load_uint8(&value) #define FT_ATOMIC_STORE_UINT8(value, new_value) \ _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ + _Py_atomic_load_uint8_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ + _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -49,7 +53,8 @@ extern "C" { _Py_atomic_store_ssize_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ _Py_atomic_store_uint8_relaxed(&value, new_value) - +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ + _Py_atomic_store_uint16_relaxed(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value #define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value @@ -62,12 +67,14 @@ extern "C" { #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value - +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #endif #ifdef __cplusplus diff --git a/Objects/genobject.c b/Objects/genobject.c index 04736a04562e14..a1ed1cbd2bfb58 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -11,6 +11,7 @@ #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -329,10 +330,11 @@ gen_close_iter(PyObject *yf) static inline bool is_resume(_Py_CODEUNIT *instr) { + uint8_t code = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); return ( - instr->op.code == RESUME || - instr->op.code == RESUME_CHECK || - instr->op.code == INSTRUMENTED_RESUME + code == RESUME || + code == RESUME_CHECK || + code == INSTRUMENTED_RESUME ); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index eee8b328a65b8c..5bb7e1211385a5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -20,7 +20,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -163,7 +163,7 @@ dummy_func( if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - this_instr->op.code = RESUME_CHECK; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } } diff --git a/Python/ceval.c b/Python/ceval.c index d130c734a67144..8b23bc6bcbeb39 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -20,7 +20,7 @@ #include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1a8554ab72269f..c88a07c1f5e951 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -162,7 +162,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* The integer overflow is checked by an assertion below. */ #define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame)))) #define NEXTOPARG() do { \ - _Py_CODEUNIT word = *next_instr; \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ opcode = word.op.code; \ oparg = word.op.arg; \ } while (0) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7c1cc147de4e1a..a5bb29385844d8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4955,7 +4955,7 @@ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - this_instr->op.code = RESUME_CHECK; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 328a3b1733d604..ce97c3add7d0ea 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -626,9 +626,10 @@ de_instrument(PyCodeObject *code, int i, int event) return; } CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - *opcode_ptr = deinstrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented); if (_PyOpcode_Caches[deinstrumented]) { - instr[1].counter = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); } } @@ -703,8 +704,10 @@ instrument(PyCodeObject *code, int i) int deopt = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); - *opcode_ptr = instrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); if (_PyOpcode_Caches[deopt]) { + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); instr[1].counter = adaptive_counter_warmup(); } }

1 0