https://github.com/python/cpython/commit/74e425ec186dde6bcfb172616fe8f35ccb5... commit: 74e425ec186dde6bcfb172616fe8f35ccb5a09bb branch: main author: Victor Stinner <vstinner@python.org> committer: vstinner <vstinner@python.org> date: 2023-09-30T19:25:54+02:00 summary: gh-110014: Fix _POSIX_THREADS and _POSIX_SEMAPHORES usage (#110139) * pycore_pythread.h is now the central place to make sure that _POSIX_THREADS and _POSIX_SEMAPHORES macros are defined if available. * Make sure that pycore_pythread.h is included when _POSIX_THREADS and _POSIX_SEMAPHORES macros are tested. * PY_TIMEOUT_MAX is now defined as a constant, since its value depends on _POSIX_THREADS, instead of being defined as a macro. * Prevent integer overflow in the preprocessor when computing PY_TIMEOUT_MAX_VALUE on Windows: replace "0xFFFFFFFELL * 1000 < LLONG_MAX" with "0xFFFFFFFELL < LLONG_MAX / 1000". * Document the change and give hints how to fix affected code. * Add an exception for PY_TIMEOUT_MAX name to smelly.py * Add PY_TIMEOUT_MAX to the stable ABI files: M Doc/data/stable_abi.dat M Doc/whatsnew/3.13.rst M Include/internal/pycore_condvar.h M Include/internal/pycore_pythread.h M Include/internal/pycore_semaphore.h M Include/pythread.h M Lib/test/test_stable_abi_ctypes.py M Misc/stable_abi.toml M PC/python3dll.c M Python/condvar.h M Python/thread.c M Python/thread_pthread.h M Tools/build/smelly.py diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index c189c78238f40..07c6d514d1954 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,4 +1,5 @@ role,name,added,ifdef_note,struct_abi_kind +var,PY_TIMEOUT_MAX,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, function,PyAIter_Check,3.10,, function,PyArg_Parse,3.2,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c9e6ca8bf8886..d6188e63dd23e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -989,6 +989,10 @@ Porting to Python 3.13 * ``Python.h`` no longer includes the ``<unistd.h>`` standard header file. If needed, it should now be included explicitly. For example, it provides the functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. + As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no + longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H`` + macros defined by ``Python.h`` can be used to decide if ``<unistd.h>`` and + ``<pthread.h>`` header files can be included. (Contributed by Victor Stinner in :gh:`108765`.) * ``Python.h`` no longer includes these standard header files: ``<time.h>``, diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 489e67d4ec4f9..34c21aaad4319 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,18 +5,8 @@ # error "this header requires Py_BUILD_CORE define" #endif -#ifndef MS_WINDOWS -# include <unistd.h> // _POSIX_THREADS -#endif +#include "pycore_pythread.h" // _POSIX_THREADS -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include <pthread.h> // _POSIX_THREADS -# endif -#endif #ifdef _POSIX_THREADS /* diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 5ec2abda91e86..98019c586bc5e 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -9,29 +9,29 @@ extern "C" { #endif -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include <pthread.h> // _POSIX_THREADS -# endif -# ifndef _POSIX_THREADS -/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then - enough of the Posix threads package is implemented to support python - threads. - - This is valid for HP-UX 11.23 running on an ia64 system. If needed, add - a check of __ia64 to verify that we're running on an ia64 system instead - of a pa-risc system. -*/ -# ifdef __hpux -# ifdef _SC_THREADS -# define _POSIX_THREADS -# endif -# endif -# endif /* _POSIX_THREADS */ -#endif /* _POSIX_THREADS */ +// Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available +#if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) +# include <unistd.h> // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if (defined(HAVE_PTHREAD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) + // This means pthreads are not implemented in libc headers, hence the macro + // not present in <unistd.h>. But they still can be implemented as an + // external library (e.g. gnu pth in pthread emulation) +# include <pthread.h> // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if !defined(_POSIX_THREADS) && defined(__hpux) && defined(_SC_THREADS) + // Check if we're running on HP-UX and _SC_THREADS is defined. If so, then + // enough of the POSIX threads package is implemented to support Python + // threads. + // + // This is valid for HP-UX 11.23 running on an ia64 system. If needed, add + // a check of __ia64 to verify that we're running on an ia64 system instead + // of a pa-risc system. +# define _POSIX_THREADS +#endif + #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) # define _USE_PTHREADS diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 2a4ecb7147ace..4c37df7b39a48 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -7,7 +7,8 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t +#include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN @@ -26,6 +27,7 @@ # include <semaphore.h> #endif + #ifdef __cplusplus extern "C" { #endif diff --git a/Include/pythread.h b/Include/pythread.h index 63714437c496b..2c2fd63d72428 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -44,22 +44,7 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); */ #define PY_TIMEOUT_T long long -#if defined(_POSIX_THREADS) - /* PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), - convert microseconds to nanoseconds. */ -# define PY_TIMEOUT_MAX (LLONG_MAX / 1000) -#elif defined (NT_THREADS) - // WaitForSingleObject() accepts timeout in milliseconds in the range - // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no - // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. -# if 0xFFFFFFFELL * 1000 < LLONG_MAX -# define PY_TIMEOUT_MAX (0xFFFFFFFELL * 1000) -# else -# define PY_TIMEOUT_MAX LLONG_MAX -# endif -#else -# define PY_TIMEOUT_MAX LLONG_MAX -#endif +PyAPI_DATA(const long long) PY_TIMEOUT_MAX; /* If microseconds == 0, the call is non-blocking: it returns immediately diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 94f817f8e1d15..6e9496d40da47 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -35,6 +35,7 @@ def test_windows_feature_macros(self): SYMBOL_NAMES = ( + "PY_TIMEOUT_MAX", "PyAIter_Check", "PyArg_Parse", "PyArg_ParseTuple", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8df3f85e61eec..46e2307614e26 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1843,6 +1843,10 @@ [function.PyThread_start_new_thread] added = '3.2' +# Not mentioned in PEP 384, was implemented as a macro in Python <= 3.12 +[data.PY_TIMEOUT_MAX] + added = '3.2' + # The following were added in PC/python3.def in Python 3.3: # 7800f75827b1be557be16f3b18f5170fbf9fae08 # 9c56409d3353b8cd4cfc19e0467bbe23fd34fc92 diff --git a/PC/python3dll.c b/PC/python3dll.c index 2c1cc8098ce85..75728c7d8057e 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -768,6 +768,7 @@ EXPORT_DATA(Py_FileSystemDefaultEncodeErrors) EXPORT_DATA(Py_FileSystemDefaultEncoding) EXPORT_DATA(Py_GenericAliasType) EXPORT_DATA(Py_HasFileSystemDefaultEncoding) +EXPORT_DATA(PY_TIMEOUT_MAX) EXPORT_DATA(Py_UTF8Mode) EXPORT_DATA(Py_Version) EXPORT_DATA(PyBaseObject_Type) diff --git a/Python/condvar.h b/Python/condvar.h index 4ddc5311cf8fa..d54db94f2c871 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -41,7 +41,8 @@ #define _CONDVAR_IMPL_H_ #include "Python.h" -#include "pycore_condvar.h" +#include "pycore_pythread.h" // _POSIX_THREADS + #ifdef _POSIX_THREADS /* diff --git a/Python/thread.c b/Python/thread.c index 1ac2db2937e37..bf207cecb9050 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -8,7 +8,7 @@ #include "Python.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_pythread.h" +#include "pycore_pythread.h" // _POSIX_THREADS #ifndef DONT_HAVE_STDIO_H # include <stdio.h> @@ -17,6 +17,26 @@ #include <stdlib.h> +// Define PY_TIMEOUT_MAX constant. +#ifdef _POSIX_THREADS + // PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), + // convert microseconds to nanoseconds. +# define PY_TIMEOUT_MAX_VALUE (LLONG_MAX / 1000) +#elif defined (NT_THREADS) + // WaitForSingleObject() accepts timeout in milliseconds in the range + // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no + // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. +# if 0xFFFFFFFELL < LLONG_MAX / 1000 +# define PY_TIMEOUT_MAX_VALUE (0xFFFFFFFELL * 1000) +# else +# define PY_TIMEOUT_MAX_VALUE LLONG_MAX +# endif +#else +# define PY_TIMEOUT_MAX_VALUE LLONG_MAX +#endif +const long long PY_TIMEOUT_MAX = PY_TIMEOUT_MAX_VALUE; + + static void PyThread__init_thread(void); /* Forward */ #define initialized _PyRuntime.threads.initialized diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index f96c57da64636..76a1f7763f23b 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,4 +1,5 @@ -#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_pythread.h" // _POSIX_SEMAPHORES /* Posix threads interface */ @@ -84,10 +85,10 @@ /* On FreeBSD 4.x, _POSIX_SEMAPHORES is defined empty, so we need to add 0 to make it work there as well. */ #if (_POSIX_SEMAPHORES+0) == -1 -#define HAVE_BROKEN_POSIX_SEMAPHORES +# define HAVE_BROKEN_POSIX_SEMAPHORES #else -#include <semaphore.h> -#include <errno.h> +# include <semaphore.h> +# include <errno.h> #endif #endif diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 276a5ab2cc84c..ab345307ff9b6 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -11,6 +11,11 @@ if sys.platform == 'darwin': ALLOWED_PREFIXES += ('__Py',) +# "Legacy": some old symbols are prefixed by "PY_". +EXCEPTIONS = frozenset({ + 'PY_TIMEOUT_MAX', +}) + IGNORED_EXTENSION = "_ctypes_test" # Ignore constructor and destructor functions IGNORED_SYMBOLS = {'_init', '_fini'} @@ -72,7 +77,7 @@ def get_smelly_symbols(stdout): symbol = parts[-1] result = '%s (type: %s)' % (symbol, symtype) - if symbol.startswith(ALLOWED_PREFIXES): + if symbol.startswith(ALLOWED_PREFIXES) or symbol in EXCEPTIONS: python_symbols.append(result) continue