Mailman 3 September 2023 - Python-checkins

gh-110152: regrtest handles cross compilation and HOSTRUNNER (#110156)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/53eb9a676f8c59b206dfc536b7590f6563… commit: 53eb9a676f8c59b206dfc536b7590f6563ad65e0 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T22:37:23Z summary: gh-110152: regrtest handles cross compilation and HOSTRUNNER (#110156) * _add_python_opts() now handles cross compilation and HOSTRUNNER. * display_header() now tells if Python is cross-compiled, display HOSTRUNNER, and get the host platform. * Remove Tools/scripts/run_tests.py script. * Remove "make hostrunnertest": use "make buildbottest" or "make test" instead. files: A Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst D Tools/scripts/run_tests.py M Lib/test/libregrtest/main.py M Lib/test/libregrtest/utils.py M Lib/test/test_regrtest.py M Makefile.pre.in diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dcb2c5870de17..19bf235845603 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -3,6 +3,7 @@ import re import shlex import sys +import sysconfig import time from test import support @@ -22,6 +23,7 @@ strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, + is_cross_compiled, get_host_runner, MS_WINDOWS, EXIT_TIMEOUT) @@ -71,10 +73,9 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks - ci_mode = (ns.fast_ci or ns.slow_ci) + self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.want_add_python_opts: bool = (_add_python_opts - and ns._add_python_opts - and ci_mode) + and ns._add_python_opts) # Select tests if ns.match_tests: @@ -431,7 +432,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if (self.want_header or not(self.pgo or self.quiet or self.single_test_run or tests or self.cmdline_args)): - display_header(self.use_resources) + display_header(self.use_resources, self.python_cmd) if self.randomize: print("Using random seed", self.random_seed) @@ -489,8 +490,56 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: # processes. return self._run_tests(selected, tests) - def _add_python_opts(self): - python_opts = [] + def _add_cross_compile_opts(self, regrtest_opts): + # WASM/WASI buildbot builders pass multiple PYTHON environment + # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. + keep_environ = bool(self.python_cmd) + environ = None + + # Are we using cross-compilation? + cross_compile = is_cross_compiled() + + # Get HOSTRUNNER + hostrunner = get_host_runner() + + if cross_compile: + # emulate -E, but keep PYTHONPATH + cross compile env vars, + # so test executable can load correct sysconfigdata file. + keep = { + '_PYTHON_PROJECT_BASE', + '_PYTHON_HOST_PLATFORM', + '_PYTHON_SYSCONFIGDATA_NAME', + 'PYTHONPATH' + } + old_environ = os.environ + new_environ = { + name: value for name, value in os.environ.items() + if not name.startswith(('PYTHON', '_PYTHON')) or name in keep + } + # Only set environ if at least one variable was removed + if new_environ != old_environ: + environ = new_environ + keep_environ = True + + if cross_compile and hostrunner: + if self.num_workers == 0: + # For now use only two cores for cross-compiled builds; + # hostrunner can be expensive. + regrtest_opts.extend(['-j', '2']) + + # If HOSTRUNNER is set and -p/--python option is not given, then + # use hostrunner to execute python binary for tests. + if not self.python_cmd: + buildpython = sysconfig.get_config_var("BUILDPYTHON") + python_cmd = f"{hostrunner} {buildpython}" + regrtest_opts.extend(["--python", python_cmd]) + keep_environ = True + + return (environ, keep_environ) + + def _add_ci_python_opts(self, python_opts, keep_environ): + # --fast-ci and --slow-ci add options to Python: + # "-u -W default -bb -E" # Unbuffered stdout and stderr if not sys.stdout.write_through: @@ -504,32 +553,27 @@ def _add_python_opts(self): if sys.flags.bytes_warning < 2: python_opts.append('-bb') - # WASM/WASI buildbot builders pass multiple PYTHON environment - # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. - if not self.python_cmd: + if not keep_environ: # Ignore PYTHON* environment variables if not sys.flags.ignore_environment: python_opts.append('-E') - if not python_opts: - return - - cmd = [*sys.orig_argv, "--dont-add-python-opts"] - cmd[1:1] = python_opts - + def _execute_python(self, cmd, environ): # Make sure that messages before execv() are logged sys.stdout.flush() sys.stderr.flush() cmd_text = shlex.join(cmd) try: + print(f"+ {cmd_text}", flush=True) + if hasattr(os, 'execv') and not MS_WINDOWS: os.execv(cmd[0], cmd) # On success, execv() do no return. # On error, it raises an OSError. else: import subprocess - with subprocess.Popen(cmd) as proc: + with subprocess.Popen(cmd, env=environ) as proc: try: proc.wait() except KeyboardInterrupt: @@ -548,6 +592,28 @@ def _add_python_opts(self): f"Command: {cmd_text}") # continue executing main() + def _add_python_opts(self): + python_opts = [] + regrtest_opts = [] + + environ, keep_environ = self._add_cross_compile_opts(regrtest_opts) + if self.ci_mode: + self._add_ci_python_opts(python_opts, keep_environ) + + if (not python_opts) and (not regrtest_opts) and (environ is None): + # Nothing changed: nothing to do + return + + # Create new command line + cmd = list(sys.orig_argv) + if python_opts: + cmd[1:1] = python_opts + if regrtest_opts: + cmd.extend(regrtest_opts) + cmd.append("--dont-add-python-opts") + + self._execute_python(cmd, environ) + def _init(self): # Set sys.stdout encoder error handler to backslashreplace, # similar to sys.stderr error handler, to avoid UnicodeEncodeError diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index dc1fa51b80dea..d2c274d997073 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -5,7 +5,9 @@ import os.path import platform import random +import shlex import signal +import subprocess import sys import sysconfig import tempfile @@ -523,7 +525,18 @@ def adjust_rlimit_nofile(): f"{new_fd_limit}: {err}.") -def display_header(use_resources: tuple[str, ...]): +def get_host_runner(): + if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: + hostrunner = sysconfig.get_config_var("HOSTRUNNER") + return hostrunner + + +def is_cross_compiled(): + return ('_PYTHON_HOST_PLATFORM' in os.environ) + + +def display_header(use_resources: tuple[str, ...], + python_cmd: tuple[str, ...] | None): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), @@ -537,13 +550,35 @@ def display_header(use_resources: tuple[str, ...]): print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) - if use_resources: print(f"== resources ({len(use_resources)}): " f"{', '.join(sorted(use_resources))}") else: print("== resources: (all disabled, use -u option)") + cross_compile = is_cross_compiled() + if cross_compile: + print("== cross compiled: Yes") + if python_cmd: + cmd = shlex.join(python_cmd) + print(f"== host python: {cmd}") + + get_cmd = [*python_cmd, '-m', 'platform'] + proc = subprocess.run( + get_cmd, + stdout=subprocess.PIPE, + text=True, + cwd=os_helper.SAVEDCWD) + stdout = proc.stdout.replace('\n', ' ').strip() + if stdout: + print(f"== host platform: {stdout}") + elif proc.returncode: + print(f"== host platform: <command failed with exit code {proc.returncode}>") + else: + hostrunner = get_host_runner() + if hostrunner: + print(f"== host runner: {hostrunner}") + # This makes it easier to remember what to set in your local # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e940cf04321d0..0e052e28ec260 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -788,14 +788,6 @@ def test_script_autotest(self): args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) - @unittest.skipUnless(sysconfig.is_python_build(), - 'run_tests.py script is not installed') - def test_tools_script_run_tests(self): - # Tools/scripts/run_tests.py - script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - args = [script, *self.regrtest_args, *self.tests] - self.run_tests(args) - def run_batch(self, *args): proc = self.run_command(args) self.check_output(proc.stdout) diff --git a/Makefile.pre.in b/Makefile.pre.in index fa5b9e6654c26..cf03c86f18b3c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1837,7 +1837,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) TESTOPTS= $(EXTRATESTOPTS) TESTPYTHON= $(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS) -TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py +TESTRUNNER= $(TESTPYTHON) -m test TESTTIMEOUT= # Remove "test_python_*" directories of previous failed test jobs. @@ -1875,11 +1875,6 @@ buildbottest: all fi $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) -# Like buildbottest, but run Python tests with HOSTRUNNER directly. -.PHONY: hostrunnertest -hostrunnertest: all - $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) - .PHONY: pythoninfo pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo diff --git a/Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst b/Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst new file mode 100644 index 0000000000000..2fb6cbbad0c44 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst @@ -0,0 +1,5 @@ +Remove ``Tools/scripts/run_tests.py`` and ``make hostrunnertest``. Just run +``./python -m test --slow-ci``, ``make buildbottest`` or ``make test`` instead. +Python test runner (regrtest) now handles cross-compilation and HOSTRUNNER. It +also adds options to Python such fast ``-u -E -W default -bb`` when +``--fast-ci`` or ``--slow-ci`` option is used. Patch by Victor Stinner. diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py deleted file mode 100644 index 3e3d15d3b0da5..0000000000000 --- a/Tools/scripts/run_tests.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Run Python's test suite in a fast, rigorous way. - -The defaults are meant to be reasonably thorough, while skipping certain -tests that can be time-consuming or resource-intensive (e.g. largefile), -or distracting (e.g. audio and gui). These defaults can be overridden by -simply passing a -u option to this script. - -""" - -import os -import shlex -import sys -import sysconfig -import test.support - - -def is_multiprocess_flag(arg): - return arg.startswith('-j') or arg.startswith('--multiprocess') - - -def is_python_flag(arg): - return arg.startswith('-p') or arg.startswith('--python') - - -def main(regrtest_args): - args = [sys.executable] - - cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ - if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: - hostrunner = sysconfig.get_config_var("HOSTRUNNER") - if cross_compile: - # emulate -E, but keep PYTHONPATH + cross compile env vars, so - # test executable can load correct sysconfigdata file. - keep = { - '_PYTHON_PROJECT_BASE', - '_PYTHON_HOST_PLATFORM', - '_PYTHON_SYSCONFIGDATA_NAME', - 'PYTHONPATH' - } - environ = { - name: value for name, value in os.environ.items() - if not name.startswith(('PYTHON', '_PYTHON')) or name in keep - } - else: - environ = os.environ.copy() - - # Allow user-specified interpreter options to override our defaults. - args.extend(test.support.args_from_interpreter_flags()) - - args.extend(['-m', 'test', # Run the test suite - '--fast-ci', # Fast Continuous Integration mode - ]) - if not any(is_multiprocess_flag(arg) for arg in regrtest_args): - if cross_compile and hostrunner: - # For now use only two cores for cross-compiled builds; - # hostrunner can be expensive. - args.extend(['-j', '2']) - - if cross_compile and hostrunner: - # If HOSTRUNNER is set and -p/--python option is not given, then - # use hostrunner to execute python binary for tests. - if not any(is_python_flag(arg) for arg in regrtest_args): - buildpython = sysconfig.get_config_var("BUILDPYTHON") - args.extend(["--python", f"{hostrunner} {buildpython}"]) - - args.extend(regrtest_args) - - print(shlex.join(args), flush=True) - - if sys.platform == 'win32': - from subprocess import call - sys.exit(call(args)) - else: - os.execve(sys.executable, args, environ) - - -if __name__ == '__main__': - main(sys.argv[1:])

1 0

gh-110014: Fix bootstrap_hash.c: remove debug code (#110161)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/d3728ddc572fff7ffcc95301bf5265717d… commit: d3728ddc572fff7ffcc95301bf5265717dbaf476 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T22:21:20Z summary: gh-110014: Fix bootstrap_hash.c: remove debug code (#110161) Oops, I commited debug code by mistake, sorry about that. files: M Python/bootstrap_hash.c diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 86a16916304ca..92f2301a012c0 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -4,9 +4,6 @@ #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _PyRuntime -#undef HAVE_GETRANDOM -#undef HAVE_GETENTROPY - #ifdef HAVE_UNISTD_H # include <unistd.h> // close() #endif

1 0

gh-109649: Add os.process_cpu_count() function (#109907)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/c81521020d643b4a5183098470ef7e6470… commit: c81521020d643b4a5183098470ef7e6470facefb branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-10-01T00:12:51+02:00 summary: gh-109649: Add os.process_cpu_count() function (#109907) * Refactor os_sched_getaffinity_impl(): move variable definitions to their first assignment. * Fix test_posix.test_sched_getaffinity(): restore the old CPU mask when the test completes! * Doc: Specify that os.cpu_count() counts *logicial* CPUs. * Doc: Specify that os.sched_getaffinity(0) is related to the calling thread. files: A Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst M Doc/library/os.rst M Doc/whatsnew/3.13.rst M Lib/os.py M Lib/test/test_os.py M Lib/test/test_posix.py M Modules/clinic/posixmodule.c.h M Modules/posixmodule.c diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 4ffd520f9ecd8..141ab0bff5b4b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5141,8 +5141,12 @@ operating system. .. function:: sched_getaffinity(pid, /) - Return the set of CPUs the process with PID *pid* (or the current process - if zero) is restricted to. + Return the set of CPUs the process with PID *pid* is restricted to. + + If *pid* is zero, return the set of CPUs the calling thread of the current + process is restricted to. + + See also the :func:`process_cpu_count` function. .. _os-path: @@ -5183,12 +5187,11 @@ Miscellaneous System Information .. function:: cpu_count() - Return the number of CPUs in the system. Returns ``None`` if undetermined. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` + Return the number of logical CPUs in the **system**. Returns ``None`` if + undetermined. + The :func:`process_cpu_count` function can be used to get the number of + logical CPUs usable by the calling thread of the **current process**. .. versionadded:: 3.4 @@ -5202,6 +5205,20 @@ Miscellaneous System Information .. availability:: Unix. +.. function:: process_cpu_count() + + Get the number of logical CPUs usable by the calling thread of the **current + process**. Returns ``None`` if undetermined. It can be less than + :func:`cpu_count` depending on the CPU affinity. + + The :func:`cpu_count` function can be used to get the number of logical CPUs + in the **system**. + + See also the :func:`sched_getaffinity` functions. + + .. versionadded:: 3.13 + + .. function:: sysconf(name, /) Return integer-valued system configuration values. If the configuration value diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 56618b9af16b9..484443a086fdd 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -163,6 +163,13 @@ opcode documented or exposed through ``dis``, and were not intended to be used externally. +os +-- + +* Add :func:`os.process_cpu_count` function to get the number of logical CPUs + usable by the calling thread of the current process. + (Contributed by Victor Stinner in :gh:`109649`.) + pathlib ------- diff --git a/Lib/os.py b/Lib/os.py index d8c9ba4b15400..35842cedf14fc 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1136,3 +1136,17 @@ def add_dll_directory(path): cookie, nt._remove_dll_directory ) + + +if _exists('sched_getaffinity'): + def process_cpu_count(): + """ + Get the number of CPUs of the current process. + + Return the number of logical CPUs usable by the calling thread of the + current process. Return None if indeterminable. + """ + return len(sched_getaffinity(0)) +else: + # Just an alias to cpu_count() (same docstring) + process_cpu_count = cpu_count diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 66aece2c4b3eb..c1a78a70c0944 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3996,14 +3996,42 @@ def test_oserror_filename(self): self.fail(f"No exception thrown by {func}") class CPUCountTests(unittest.TestCase): + def check_cpu_count(self, cpus): + if cpus is None: + self.skipTest("Could not determine the number of CPUs") + + self.assertIsInstance(cpus, int) + self.assertGreater(cpus, 0) + def test_cpu_count(self): cpus = os.cpu_count() - if cpus is not None: - self.assertIsInstance(cpus, int) - self.assertGreater(cpus, 0) - else: + self.check_cpu_count(cpus) + + def test_process_cpu_count(self): + cpus = os.process_cpu_count() + self.assertLessEqual(cpus, os.cpu_count()) + self.check_cpu_count(cpus) + + @unittest.skipUnless(hasattr(os, 'sched_setaffinity'), + "don't have sched affinity support") + def test_process_cpu_count_affinity(self): + ncpu = os.cpu_count() + if ncpu is None: self.skipTest("Could not determine the number of CPUs") + # Disable one CPU + mask = os.sched_getaffinity(0) + if len(mask) <= 1: + self.skipTest(f"sched_getaffinity() returns less than " + f"2 CPUs: {sorted(mask)}") + self.addCleanup(os.sched_setaffinity, 0, list(mask)) + mask.pop() + os.sched_setaffinity(0, mask) + + # test process_cpu_count() + affinity = os.process_cpu_count() + self.assertEqual(affinity, ncpu - 1) + # FD inheritance check is only useful for systems with process support. @support.requires_subprocess() diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 444f8abe4607b..9d72dba159c6b 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1205,6 +1205,7 @@ def test_sched_getaffinity(self): @requires_sched_affinity def test_sched_setaffinity(self): mask = posix.sched_getaffinity(0) + self.addCleanup(posix.sched_setaffinity, 0, list(mask)) if len(mask) > 1: # Empty masks are forbidden mask.pop() diff --git a/Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst b/Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst new file mode 100644 index 0000000000000..ab708e6fb9a7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst @@ -0,0 +1,2 @@ +Add :func:`os.process_cpu_count` function to get the number of logical CPUs +usable by the calling thread of the current process. Patch by Victor Stinner. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index e77a31b947f45..fc39ab72bf2a5 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10425,11 +10425,9 @@ PyDoc_STRVAR(os_cpu_count__doc__, "cpu_count($module, /)\n" "--\n" "\n" -"Return the number of CPUs in the system; return None if indeterminable.\n" +"Return the number of logical CPUs in the system.\n" "\n" -"This number is not equivalent to the number of CPUs the current process can\n" -"use. The number of usable CPUs can be obtained with\n" -"``len(os.sched_getaffinity(0))``"); +"Return None if indeterminable."); #define OS_CPU_COUNT_METHODDEF \ {"cpu_count", (PyCFunction)os_cpu_count, METH_NOARGS, os_cpu_count__doc__}, @@ -11988,4 +11986,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=51aa26bc6a41e1da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8b60de6ddb925bc3 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0b252092573e5..d3c0aa6f3c538 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8133,39 +8133,45 @@ static PyObject * os_sched_getaffinity_impl(PyObject *module, pid_t pid) /*[clinic end generated code: output=f726f2c193c17a4f input=983ce7cb4a565980]*/ { - int cpu, ncpus, count; + int ncpus = NCPUS_START; size_t setsize; - cpu_set_t *mask = NULL; - PyObject *res = NULL; + cpu_set_t *mask; - ncpus = NCPUS_START; while (1) { setsize = CPU_ALLOC_SIZE(ncpus); mask = CPU_ALLOC(ncpus); - if (mask == NULL) + if (mask == NULL) { return PyErr_NoMemory(); - if (sched_getaffinity(pid, setsize, mask) == 0) + } + if (sched_getaffinity(pid, setsize, mask) == 0) { break; + } CPU_FREE(mask); - if (errno != EINVAL) + if (errno != EINVAL) { return posix_error(); + } if (ncpus > INT_MAX / 2) { - PyErr_SetString(PyExc_OverflowError, "could not allocate " - "a large enough CPU set"); + PyErr_SetString(PyExc_OverflowError, + "could not allocate a large enough CPU set"); return NULL; } - ncpus = ncpus * 2; + ncpus *= 2; } - res = PySet_New(NULL); - if (res == NULL) + PyObject *res = PySet_New(NULL); + if (res == NULL) { goto error; - for (cpu = 0, count = CPU_COUNT_S(setsize, mask); count; cpu++) { + } + + int cpu = 0; + int count = CPU_COUNT_S(setsize, mask); + for (; count; cpu++) { if (CPU_ISSET_S(cpu, setsize, mask)) { PyObject *cpu_num = PyLong_FromLong(cpu); --count; - if (cpu_num == NULL) + if (cpu_num == NULL) { goto error; + } if (PySet_Add(res, cpu_num)) { Py_DECREF(cpu_num); goto error; @@ -8177,12 +8183,12 @@ os_sched_getaffinity_impl(PyObject *module, pid_t pid) return res; error: - if (mask) + if (mask) { CPU_FREE(mask); + } Py_XDECREF(res); return NULL; } - #endif /* HAVE_SCHED_SETAFFINITY */ #endif /* HAVE_SCHED_H */ @@ -14333,44 +14339,49 @@ os_get_terminal_size_impl(PyObject *module, int fd) /*[clinic input] os.cpu_count -Return the number of CPUs in the system; return None if indeterminable. +Return the number of logical CPUs in the system. -This number is not equivalent to the number of CPUs the current process can -use. The number of usable CPUs can be obtained with -``len(os.sched_getaffinity(0))`` +Return None if indeterminable. [clinic start generated code]*/ static PyObject * os_cpu_count_impl(PyObject *module) -/*[clinic end generated code: output=5fc29463c3936a9c input=e7c8f4ba6dbbadd3]*/ +/*[clinic end generated code: output=5fc29463c3936a9c input=ba2f6f8980a0e2eb]*/ { - int ncpu = 0; + int ncpu; #ifdef MS_WINDOWS -#ifdef MS_WINDOWS_DESKTOP +# ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); -#endif +# else + ncpu = 0; +# endif + #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); + #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) ncpu = sysconf(_SC_NPROCESSORS_ONLN); + #elif defined(__VXWORKS__) ncpu = _Py_popcount32(vxCpuEnabledGet()); + #elif defined(__DragonFly__) || \ defined(__OpenBSD__) || \ defined(__FreeBSD__) || \ defined(__NetBSD__) || \ defined(__APPLE__) - int mib[2]; + ncpu = 0; size_t len = sizeof(ncpu); - mib[0] = CTL_HW; - mib[1] = HW_NCPU; - if (sysctl(mib, 2, &ncpu, &len, NULL, 0) != 0) + int mib[2] = {CTL_HW, HW_NCPU}; + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) != 0) { ncpu = 0; + } #endif - if (ncpu >= 1) - return PyLong_FromLong(ncpu); - else + + if (ncpu < 1) { Py_RETURN_NONE; + } + return PyLong_FromLong(ncpu); }

1 0

[3.11] gh-110088: Fix asyncio test_prompt_cancellation() (GH-110157) (#110159)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/446c81c3c3a5e82c1df475b8a98009aeb5… commit: 446c81c3c3a5e82c1df475b8a98009aeb5b5433e branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T21:01:33Z summary: [3.11] gh-110088: Fix asyncio test_prompt_cancellation() (GH-110157) (#110159) gh-110088: Fix asyncio test_prompt_cancellation() (GH-110157) Don't measure the CI performance: don't test the maximum elapsed time. The check failed on a slow CI. (cherry picked from commit c62b49ecc8da13fa9522865ef6fe0aec194fd0d8) Co-authored-by: Victor Stinner <vstinner(a)python.org> files: M Lib/test/test_asyncio/test_events.py diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index daf2fb87994a8..d7871d3e538cb 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1671,12 +1671,9 @@ async def main(): self.loop.stop() return res - start = time.monotonic() t = self.loop.create_task(main()) self.loop.run_forever() - elapsed = time.monotonic() - start - self.assertLess(elapsed, 0.1) self.assertEqual(t.result(), 'cancelled') self.assertRaises(asyncio.CancelledError, f.result) if ov is not None:

1 0

gh-109276: regrtest: add WORKER_FAILED state (#110148)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/2c234196ea30b9da370780204ed9068f1f… commit: 2c234196ea30b9da370780204ed9068f1fb134c6 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T22:48:26+02:00 summary: gh-109276: regrtest: add WORKER_FAILED state (#110148) Rename WORKER_ERROR to WORKER_BUG. Add WORKER_FAILED state: it does not stop the manager, whereas WORKER_BUG does. Change also TestResults.display_result() order: display failed tests at the end, the important important information. WorkerThread now tries to get the signal name for negative exit code. files: M Lib/test/libregrtest/result.py M Lib/test/libregrtest/results.py M Lib/test/libregrtest/run_workers.py M Lib/test/libregrtest/utils.py M Lib/test/test_regrtest.py diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index bf885264657d5..d6b0d5ad383a5 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -19,7 +19,8 @@ class State: ENV_CHANGED = "ENV_CHANGED" RESOURCE_DENIED = "RESOURCE_DENIED" INTERRUPTED = "INTERRUPTED" - MULTIPROCESSING_ERROR = "MULTIPROCESSING_ERROR" + WORKER_FAILED = "WORKER_FAILED" # non-zero worker process exit code + WORKER_BUG = "WORKER_BUG" # exception when running a worker DID_NOT_RUN = "DID_NOT_RUN" TIMEOUT = "TIMEOUT" @@ -29,7 +30,8 @@ def is_failed(state): State.FAILED, State.UNCAUGHT_EXC, State.REFLEAK, - State.MULTIPROCESSING_ERROR, + State.WORKER_FAILED, + State.WORKER_BUG, State.TIMEOUT} @staticmethod @@ -42,14 +44,16 @@ def has_meaningful_duration(state): State.SKIPPED, State.RESOURCE_DENIED, State.INTERRUPTED, - State.MULTIPROCESSING_ERROR, + State.WORKER_FAILED, + State.WORKER_BUG, State.DID_NOT_RUN} @staticmethod def must_stop(state): return state in { State.INTERRUPTED, - State.MULTIPROCESSING_ERROR} + State.WORKER_BUG, + } @dataclasses.dataclass(slots=True) @@ -108,8 +112,10 @@ def __str__(self) -> str: return f"{self.test_name} skipped (resource denied)" case State.INTERRUPTED: return f"{self.test_name} interrupted" - case State.MULTIPROCESSING_ERROR: - return f"{self.test_name} process crashed" + case State.WORKER_FAILED: + return f"{self.test_name} worker non-zero exit code" + case State.WORKER_BUG: + return f"{self.test_name} worker bug" case State.DID_NOT_RUN: return f"{self.test_name} ran no tests" case State.TIMEOUT: diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 35df50d581ff6..3708078ff0bf3 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -30,6 +30,7 @@ def __init__(self): self.rerun_results: list[TestResult] = [] self.interrupted: bool = False + self.worker_bug: bool = False self.test_times: list[tuple[float, TestName]] = [] self.stats = TestStats() # used by --junit-xml @@ -38,7 +39,8 @@ def __init__(self): def is_all_good(self): return (not self.bad and not self.skipped - and not self.interrupted) + and not self.interrupted + and not self.worker_bug) def get_executed(self): return (set(self.good) | set(self.bad) | set(self.skipped) @@ -60,6 +62,8 @@ def get_state(self, fail_env_changed): if self.interrupted: state.append("INTERRUPTED") + if self.worker_bug: + state.append("WORKER BUG") if not state: state.append("SUCCESS") @@ -77,6 +81,8 @@ def get_exitcode(self, fail_env_changed, fail_rerun): exitcode = EXITCODE_NO_TESTS_RAN elif fail_rerun and self.rerun: exitcode = EXITCODE_RERUN_FAIL + elif self.worker_bug: + exitcode = EXITCODE_BAD_TEST return exitcode def accumulate_result(self, result: TestResult, runtests: RunTests): @@ -105,6 +111,9 @@ def accumulate_result(self, result: TestResult, runtests: RunTests): else: raise ValueError(f"invalid test state: {result.state!r}") + if result.state == State.WORKER_BUG: + self.worker_bug = True + if result.has_meaningful_duration() and not rerun: self.test_times.append((result.duration, test_name)) if result.stats is not None: @@ -173,12 +182,6 @@ def write_junit(self, filename: StrPath): f.write(s) def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): - omitted = set(tests) - self.get_executed() - if omitted: - print() - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if print_slowest: self.test_times.sort(reverse=True) print() @@ -186,16 +189,21 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): for test_time, test in self.test_times[:10]: print("- %s: %s" % (test, format_duration(test_time))) - all_tests = [ - (self.bad, "test", "{} failed:"), - (self.env_changed, "test", "{} altered the execution environment (env changed):"), - ] + all_tests = [] + omitted = set(tests) - self.get_executed() + + # less important + all_tests.append((omitted, "test", "{} omitted:")) if not quiet: all_tests.append((self.skipped, "test", "{} skipped:")) all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) - all_tests.append((self.rerun, "re-run test", "{}:")) all_tests.append((self.run_no_tests, "test", "{} run no tests:")) + # more important + all_tests.append((self.env_changed, "test", "{} altered the execution environment (env changed):")) + all_tests.append((self.rerun, "re-run test", "{}:")) + all_tests.append((self.bad, "test", "{} failed:")) + for tests_list, count_text, title_format in all_tests: if tests_list: print() diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 41ed7b0bac01a..6eb32e5963586 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -22,7 +22,7 @@ from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, MS_WINDOWS, - format_duration, print_warning, count, plural) + format_duration, print_warning, count, plural, get_signal_name) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -92,7 +92,7 @@ def __init__(self, test_name: TestName, err_msg: str | None, stdout: str | None, - state: str = State.MULTIPROCESSING_ERROR): + state: str): result = TestResult(test_name, state=state) self.mp_result = MultiprocessResult(result, stdout, err_msg) super().__init__() @@ -298,7 +298,9 @@ def read_stdout(self, stdout_file: TextIO) -> str: # gh-101634: Catch UnicodeDecodeError if stdout cannot be # decoded from encoding raise WorkerError(self.test_name, - f"Cannot read process stdout: {exc}", None) + f"Cannot read process stdout: {exc}", + stdout=None, + state=State.WORKER_BUG) def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, stdout: str) -> tuple[TestResult, str]: @@ -317,10 +319,11 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, # decoded from encoding err_msg = f"Failed to read worker process JSON: {exc}" raise WorkerError(self.test_name, err_msg, stdout, - state=State.MULTIPROCESSING_ERROR) + state=State.WORKER_BUG) if not worker_json: - raise WorkerError(self.test_name, "empty JSON", stdout) + raise WorkerError(self.test_name, "empty JSON", stdout, + state=State.WORKER_BUG) try: result = TestResult.from_json(worker_json) @@ -329,7 +332,7 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, # decoded from encoding err_msg = f"Failed to parse worker process JSON: {exc}" raise WorkerError(self.test_name, err_msg, stdout, - state=State.MULTIPROCESSING_ERROR) + state=State.WORKER_BUG) return (result, stdout) @@ -345,9 +348,15 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: stdout = self.read_stdout(stdout_file) if retcode is None: - raise WorkerError(self.test_name, None, stdout, state=State.TIMEOUT) + raise WorkerError(self.test_name, stdout=stdout, + err_msg=None, + state=State.TIMEOUT) if retcode != 0: - raise WorkerError(self.test_name, f"Exit code {retcode}", stdout) + name = get_signal_name(retcode) + if name: + retcode = f"{retcode} ({name})" + raise WorkerError(self.test_name, f"Exit code {retcode}", stdout, + state=State.WORKER_FAILED) result, stdout = self.read_json(json_file, json_tmpfile, stdout) @@ -527,7 +536,7 @@ def display_result(self, mp_result: MultiprocessResult) -> None: text = str(result) if mp_result.err_msg: - # MULTIPROCESSING_ERROR + # WORKER_BUG text += ' (%s)' % mp_result.err_msg elif (result.duration >= PROGRESS_MIN_TIME and not pgo): text += ' (%s)' % format_duration(result.duration) @@ -543,7 +552,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: # Thread got an exception format_exc = item[1] print_warning(f"regrtest worker thread failed: {format_exc}") - result = TestResult("<regrtest worker>", state=State.MULTIPROCESSING_ERROR) + result = TestResult("<regrtest worker>", state=State.WORKER_BUG) self.results.accumulate_result(result, self.runtests) return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 46451152b8859..dc1fa51b80dea 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -5,6 +5,7 @@ import os.path import platform import random +import signal import sys import sysconfig import tempfile @@ -581,3 +582,24 @@ def cleanup_temp_dir(tmp_dir: StrPath): else: print("Remove file: %s" % name) os_helper.unlink(name) + +WINDOWS_STATUS = { + 0xC0000005: "STATUS_ACCESS_VIOLATION", + 0xC00000FD: "STATUS_STACK_OVERFLOW", + 0xC000013A: "STATUS_CONTROL_C_EXIT", +} + +def get_signal_name(exitcode): + if exitcode < 0: + signum = -exitcode + try: + return signal.Signals(signum).name + except ValueError: + pass + + try: + return WINDOWS_STATUS[exitcode] + except KeyError: + pass + + return None diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c98b05abcea98..e940cf04321d0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -14,6 +14,7 @@ import random import re import shlex +import signal import subprocess import sys import sysconfig @@ -2066,6 +2067,15 @@ def test_normalize_test_name(self): self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) + def test_get_signal_name(self): + for exitcode, expected in ( + (-int(signal.SIGINT), 'SIGINT'), + (-int(signal.SIGSEGV), 'SIGSEGV'), + (3221225477, "STATUS_ACCESS_VIOLATION"), + (0xC00000FD, "STATUS_STACK_OVERFLOW"), + ): + self.assertEqual(utils.get_signal_name(exitcode), expected, exitcode) + if __name__ == '__main__': unittest.main()

1 0

gh-110088: Fix asyncio test_prompt_cancellation() (#110157)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/c62b49ecc8da13fa9522865ef6fe0aec19… commit: c62b49ecc8da13fa9522865ef6fe0aec194fd0d8 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T22:40:10+02:00 summary: gh-110088: Fix asyncio test_prompt_cancellation() (#110157) Don't measure the CI performance: don't test the maximum elapsed time. The check failed on a slow CI. files: M Lib/test/test_asyncio/test_events.py diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 3ee6565b2b65a..b25c0975736e2 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1693,12 +1693,9 @@ async def main(): self.loop.stop() return res - start = time.monotonic() t = self.loop.create_task(main()) self.loop.run_forever() - elapsed = time.monotonic() - start - self.assertLess(elapsed, 0.1) self.assertEqual(t.result(), 'cancelled') self.assertRaises(asyncio.CancelledError, f.result) if ov is not None:

1 0

gh-110014: Include explicitly <unistd.h> header (#110155)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/7513994c927857679544449392744be308… commit: 7513994c927857679544449392744be308d36586 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T20:06:45Z summary: gh-110014: Include explicitly <unistd.h> header (#110155) * Remove unused <locale.h> includes. * Remove unused <fcntl.h> include in traceback.h. * Remove redundant <assert.h> and <stddef.h> includes. They are already included by "Python.h". * Remove <object.h> include in faulthandler.c. Python.h already includes it. * Add missing <stdbool.h> in pycore_pythread.h if HAVE_PTHREAD_STUBS is defined. * Fix also warnings in pthread_stubs.h: don't redefine macros if they are already defined, like the __NEED_pthread_t macro. files: M Doc/whatsnew/3.13.rst M Include/cpython/pthread_stubs.h M Include/internal/pycore_pythread.h M Modules/_io/fileio.c M Modules/_randommodule.c M Modules/faulthandler.c M Modules/posixmodule.c M Objects/fileobject.c M Parser/myreadline.c M Parser/tokenizer.c M Python/bltinmodule.c M Python/bootstrap_hash.c M Python/fileutils.c M Python/frozenmain.c M Python/pylifecycle.c M Python/sysmodule.c M Python/traceback.c diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d6188e63dd23e..56618b9af16b9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -988,7 +988,8 @@ Porting to Python 3.13 * ``Python.h`` no longer includes the ``<unistd.h>`` standard header file. If needed, it should now be included explicitly. For example, it provides the - functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. + functions: ``read()``, ``write()``, ``close()``, ``isatty()``, ``lseek()``, + ``getpid()``, ``getcwd()``, ``sysconf()`` and ``getpagesize()``. As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H`` macros defined by ``Python.h`` can be used to decide if ``<unistd.h>`` and diff --git a/Include/cpython/pthread_stubs.h b/Include/cpython/pthread_stubs.h index d95ee03d8308c..5246968ea0547 100644 --- a/Include/cpython/pthread_stubs.h +++ b/Include/cpython/pthread_stubs.h @@ -21,13 +21,29 @@ #ifdef __wasi__ // WASI's bits/alltypes.h provides type definitions when __NEED_ is set. // The header file can be included multiple times. -# define __NEED_pthread_cond_t 1 -# define __NEED_pthread_condattr_t 1 -# define __NEED_pthread_mutex_t 1 -# define __NEED_pthread_mutexattr_t 1 -# define __NEED_pthread_key_t 1 -# define __NEED_pthread_t 1 -# define __NEED_pthread_attr_t 1 +// +// <sys/types.h> may also define these macros. +# ifndef __NEED_pthread_cond_t +# define __NEED_pthread_cond_t 1 +# endif +# ifndef __NEED_pthread_condattr_t +# define __NEED_pthread_condattr_t 1 +# endif +# ifndef __NEED_pthread_mutex_t +# define __NEED_pthread_mutex_t 1 +# endif +# ifndef __NEED_pthread_mutexattr_t +# define __NEED_pthread_mutexattr_t 1 +# endif +# ifndef __NEED_pthread_key_t +# define __NEED_pthread_key_t 1 +# endif +# ifndef __NEED_pthread_t +# define __NEED_pthread_t 1 +# endif +# ifndef __NEED_pthread_attr_t +# define __NEED_pthread_attr_t 1 +# endif # include <bits/alltypes.h> #else typedef struct { void *__x; } pthread_cond_t; diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 98019c586bc5e..f679c1bdb7549 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - // Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available #if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ && !defined(_POSIX_SEMAPHORES)) @@ -44,6 +43,8 @@ extern "C" { #if defined(HAVE_PTHREAD_STUBS) +#include <stdbool.h> // bool + // pthread_key struct py_stub_tls_entry { bool in_use; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index fb416700e2252..8a73ea0365b7a 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -5,20 +5,23 @@ #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include <stdbool.h> +#include <stdbool.h> // bool +#ifdef HAVE_UNISTD_H +# include <unistd.h> // lseek() +#endif #ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> +# include <sys/types.h> #endif #ifdef HAVE_SYS_STAT_H -#include <sys/stat.h> +# include <sys/stat.h> #endif #ifdef HAVE_IO_H -#include <io.h> +# include <io.h> #endif #ifdef HAVE_FCNTL_H -#include <fcntl.h> +# include <fcntl.h> // open() #endif -#include <stddef.h> /* For offsetof */ + #include "_iomodule.h" /* @@ -35,22 +38,23 @@ */ #ifdef MS_WINDOWS -/* can simulate truncate with Win32 API functions; see file_truncate */ -#define HAVE_FTRUNCATE -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include <windows.h> + // can simulate truncate with Win32 API functions; see file_truncate +# define HAVE_FTRUNCATE +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include <windows.h> #endif #if BUFSIZ < (8*1024) -#define SMALLCHUNK (8*1024) +# define SMALLCHUNK (8*1024) #elif (BUFSIZ >= (2 << 25)) -#error "unreasonable BUFSIZ > 64 MiB defined" +# error "unreasonable BUFSIZ > 64 MiB defined" #else -#define SMALLCHUNK BUFSIZ +# define SMALLCHUNK BUFSIZ #endif + /*[clinic input] module _io class _io.FileIO "fileio *" "clinic_state()->PyFileIO_Type" diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 18811d03adb45..d41093c880647 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -74,12 +74,15 @@ #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() + +#ifdef HAVE_UNISTD_H +# include <unistd.h> // getpid() +#endif #ifdef HAVE_PROCESS_H # include <process.h> // getpid() #endif - #ifdef MS_WINDOWS -# include <windows.h> +# include <windows.h> // GetCurrentProcessId() #endif /* Period parameters -- These are all magic. Don't change. */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 4b6bf68be0720..a2e3c2300b3ce 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -6,8 +6,10 @@ #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DumpTracebackThreads -#include <object.h> -#include <signal.h> +#ifdef HAVE_UNISTD_H +# include <unistd.h> // _exit() +#endif +#include <signal.h> // sigaction() #include <stdlib.h> // abort() #if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) && defined(HAVE_PTHREAD_H) # include <pthread.h> @@ -16,7 +18,7 @@ # include <windows.h> #endif #ifdef HAVE_SYS_RESOURCE_H -# include <sys/resource.h> +# include <sys/resource.h> // setrlimit() #endif #if defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_LINUX_AUXVEC_H) && defined(HAVE_SYS_AUXV_H) @@ -24,6 +26,7 @@ # include <sys/auxv.h> // getauxval() #endif + /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d7d3e365d2c55..0b252092573e5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -24,6 +24,10 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG +#ifdef HAVE_UNISTD_H +# include <unistd.h> // symlink() +#endif + #ifdef MS_WINDOWS # include <windows.h> # if !defined(MS_WINDOWS_GAMES) || defined(MS_WINDOWS_DESKTOP) @@ -37,7 +41,6 @@ # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ #endif - #ifndef MS_WINDOWS # include "posixmodule.h" #else @@ -285,10 +288,6 @@ corresponding Unix manual entries for more information on calls."); # include <sched.h> #endif -#ifdef HAVE_COPY_FILE_RANGE -# include <unistd.h> // copy_file_range() -#endif - #if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY) # undef HAVE_SCHED_SETAFFINITY #endif diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 066172baf9f02..5522eba34eace 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -4,15 +4,19 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_runtime.h" // _PyRuntime +#ifdef HAVE_UNISTD_H +# include <unistd.h> // isatty() +#endif + #if defined(HAVE_GETC_UNLOCKED) && !defined(_Py_MEMORY_SANITIZER) -/* clang MemorySanitizer doesn't yet understand getc_unlocked. */ -#define GETC(f) getc_unlocked(f) -#define FLOCKFILE(f) flockfile(f) -#define FUNLOCKFILE(f) funlockfile(f) + /* clang MemorySanitizer doesn't yet understand getc_unlocked. */ +# define GETC(f) getc_unlocked(f) +# define FLOCKFILE(f) flockfile(f) +# define FUNLOCKFILE(f) funlockfile(f) #else -#define GETC(f) getc(f) -#define FLOCKFILE(f) -#define FUNLOCKFILE(f) +# define GETC(f) getc(f) +# define FLOCKFILE(f) +# define FUNLOCKFILE(f) #endif /* Newline flags */ diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 815387388218c..719a178f244a2 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -14,11 +14,15 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #ifdef MS_WINDOWS # ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN # endif # include "windows.h" #endif /* MS_WINDOWS */ +#ifdef HAVE_UNISTD_H +# include <unistd.h> // isatty() +#endif + // Export the symbol since it's used by the readline shared extension PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 46b7159ff0516..41d0d16a471dd 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -4,10 +4,12 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() -#include <assert.h> +#include "tokenizer.h" // struct tok_state +#include "errcode.h" // E_OK -#include "tokenizer.h" -#include "errcode.h" +#ifdef HAVE_UNISTD_H +# include <unistd.h> // read() +#endif /* Alternate tab spacing */ #define ALTTABSIZE 1 diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 69056bf23f405..c373585c0986c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -17,6 +17,11 @@ #include "clinic/bltinmodule.c.h" +#ifdef HAVE_UNISTD_H +# include <unistd.h> // isatty() +#endif + + static PyObject* update_bases(PyObject *bases, PyObject *const *args, Py_ssize_t nargs) { diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index ef693e5df1fcc..86a16916304ca 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -4,22 +4,28 @@ #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _PyRuntime +#undef HAVE_GETRANDOM +#undef HAVE_GETENTROPY + +#ifdef HAVE_UNISTD_H +# include <unistd.h> // close() +#endif #ifdef MS_WINDOWS # include <windows.h> # include <bcrypt.h> #else -# include <fcntl.h> +# include <fcntl.h> // O_RDONLY # ifdef HAVE_SYS_STAT_H # include <sys/stat.h> # endif # ifdef HAVE_LINUX_RANDOM_H -# include <linux/random.h> +# include <linux/random.h> // GRND_NONBLOCK # endif # if defined(HAVE_SYS_RANDOM_H) && (defined(HAVE_GETRANDOM) || defined(HAVE_GETENTROPY)) -# include <sys/random.h> +# include <sys/random.h> // getrandom() # endif # if !defined(HAVE_GETRANDOM) && defined(HAVE_GETRANDOM_SYSCALL) -# include <sys/syscall.h> +# include <sys/syscall.h> // SYS_getrandom # endif #endif diff --git a/Python/fileutils.c b/Python/fileutils.c index 9bc1de2db8400..17a4ae56ef052 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2,8 +2,11 @@ #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "osdefs.h" // SEP -#include <locale.h> + #include <stdlib.h> // mbstowcs() +#ifdef HAVE_UNISTD_H +# include <unistd.h> // getcwd() +#endif #ifdef MS_WINDOWS # include <malloc.h> @@ -19,7 +22,7 @@ extern int winerror_to_errno(int); #endif #ifdef HAVE_LANGINFO_H -#include <langinfo.h> +# include <langinfo.h> // nl_langinfo(CODESET) #endif #ifdef HAVE_SYS_IOCTL_H @@ -27,12 +30,12 @@ extern int winerror_to_errno(int); #endif #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION -#include <iconv.h> +# include <iconv.h> // iconv_open() #endif #ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif /* HAVE_FCNTL_H */ +# include <fcntl.h> // fcntl(F_GETFD) +#endif #ifdef O_CLOEXEC /* Does open() support the O_CLOEXEC flag? Possible values: diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 767f9804903a9..3ce9476c9ad46 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -3,7 +3,11 @@ #include "Python.h" #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_runtime.h" // _PyRuntime_Initialize() -#include <locale.h> + +#ifdef HAVE_UNISTD_H +# include <unistd.h> // isatty() +#endif + #ifdef MS_WINDOWS extern void PyWinFreeze_ExeInit(void); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 23f66ec3601df..f3ed77e516237 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -37,6 +37,9 @@ #include <locale.h> // setlocale() #include <stdlib.h> // getenv() +#ifdef HAVE_UNISTD_H +# include <unistd.h> // isatty() +#endif #if defined(__APPLE__) # include <mach-o/loader.h> diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 7ba7be10aacb9..b00301765e189 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -40,7 +40,9 @@ Data members: #include "osdefs.h" // DELIM #include "stdlib_module_names.h" // _Py_stdlib_module_names -#include <locale.h> +#ifdef HAVE_UNISTD_H +# include <unistd.h> // getpid() +#endif #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN diff --git a/Python/traceback.c b/Python/traceback.c index 7e791d0a59bd8..5de1bff9943c6 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -20,13 +20,14 @@ #include "frameobject.h" // PyFrame_New() #include "osdefs.h" // SEP -#ifdef HAVE_FCNTL_H -# include <fcntl.h> +#ifdef HAVE_UNISTD_H +# include <unistd.h> // lseek() #endif -#define OFF(x) offsetof(PyTracebackObject, x) +#define OFF(x) offsetof(PyTracebackObject, x) #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) + #define MAX_STRING_LENGTH 500 #define MAX_FRAME_DEPTH 100 #define MAX_NTHREADS 100

1 0

[3.11] gh-109748: Fix again venv test_zippath_from_non_installed_posix() (GH-110149) (#110153)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/080f5b98d1eac67abc13fe9654ff8a3fee… commit: 080f5b98d1eac67abc13fe9654ff8a3feeef55ea branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T18:46:37Z summary: [3.11] gh-109748: Fix again venv test_zippath_from_non_installed_posix() (GH-110149) (#110153) gh-109748: Fix again venv test_zippath_from_non_installed_posix() (GH-110149) Call also copy_python_src_ignore() on listdir() names. shutil.copytree(): replace set() with an empty tuple. An empty tuple becomes a constant in the compiler and checking if an item is in an empty tuple is cheap. (cherry picked from commit 0def8c712bb6f66f1081cab71deb3681566b846d) Co-authored-by: Victor Stinner <vstinner(a)python.org> files: M Lib/shutil.py M Lib/test/test_support.py M Lib/test/test_venv.py diff --git a/Lib/shutil.py b/Lib/shutil.py index 1c3a75da55ba5..d108986d13026 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -454,7 +454,7 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, if ignore is not None: ignored_names = ignore(os.fspath(src), [x.name for x in entries]) else: - ignored_names = set() + ignored_names = () os.makedirs(dst, exist_ok=dirs_exist_ok) errors = [] diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 2efdbd22d90e2..01ba88ce42c5f 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -818,7 +818,7 @@ def test_copy_python_src_ignore(self): self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored | {'build', 'venv'}) - # An other directory + # Another directory path = os.path.join(src_dir, 'Objects') self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 884ac3ac567c0..eb9227a3b7037 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -572,7 +572,11 @@ def test_zippath_from_non_installed_posix(self): eachpath, os.path.join(non_installed_dir, platlibdir)) elif os.path.isfile(os.path.join(eachpath, "os.py")): - for name in os.listdir(eachpath): + names = os.listdir(eachpath) + ignored_names = copy_python_src_ignore(eachpath, names) + for name in names: + if name in ignored_names: + continue if name == "site-packages": continue fn = os.path.join(eachpath, name)

1 0

gh-109748: Fix again venv test_zippath_from_non_installed_posix() (#110149)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/0def8c712bb6f66f1081cab71deb368156… commit: 0def8c712bb6f66f1081cab71deb3681566b846d branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T20:23:26+02:00 summary: gh-109748: Fix again venv test_zippath_from_non_installed_posix() (#110149) Call also copy_python_src_ignore() on listdir() names. shutil.copytree(): replace set() with an empty tuple. An empty tuple becomes a constant in the compiler and checking if an item is in an empty tuple is cheap. files: M Lib/shutil.py M Lib/test/test_support.py M Lib/test/test_venv.py diff --git a/Lib/shutil.py b/Lib/shutil.py index b37bd082eee0c..b903f13d8b76a 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -481,7 +481,7 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, if ignore is not None: ignored_names = ignore(os.fspath(src), [x.name for x in entries]) else: - ignored_names = set() + ignored_names = () os.makedirs(dst, exist_ok=dirs_exist_ok) errors = [] diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 902bec7845130..97de81677b10b 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -832,7 +832,7 @@ def test_copy_python_src_ignore(self): self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored | {'build', 'venv'}) - # An other directory + # Another directory path = os.path.join(src_dir, 'Objects') self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 0ffe3e1d0cc49..890672c5d27ee 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -569,7 +569,11 @@ def test_zippath_from_non_installed_posix(self): eachpath, os.path.join(non_installed_dir, platlibdir)) elif os.path.isfile(os.path.join(eachpath, "os.py")): - for name in os.listdir(eachpath): + names = os.listdir(eachpath) + ignored_names = copy_python_src_ignore(eachpath, names) + for name in names: + if name in ignored_names: + continue if name == "site-packages": continue fn = os.path.join(eachpath, name)

1 0

gh-110014: Fix _POSIX_THREADS and _POSIX_SEMAPHORES usage (#110139)
by vstinner 30 Sep '23

30 Sep '23

https://github.com/python/cpython/commit/74e425ec186dde6bcfb172616fe8f35ccb… commit: 74e425ec186dde6bcfb172616fe8f35ccb5a09bb branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T19:25:54+02:00 summary: gh-110014: Fix _POSIX_THREADS and _POSIX_SEMAPHORES usage (#110139) * pycore_pythread.h is now the central place to make sure that _POSIX_THREADS and _POSIX_SEMAPHORES macros are defined if available. * Make sure that pycore_pythread.h is included when _POSIX_THREADS and _POSIX_SEMAPHORES macros are tested. * PY_TIMEOUT_MAX is now defined as a constant, since its value depends on _POSIX_THREADS, instead of being defined as a macro. * Prevent integer overflow in the preprocessor when computing PY_TIMEOUT_MAX_VALUE on Windows: replace "0xFFFFFFFELL * 1000 < LLONG_MAX" with "0xFFFFFFFELL < LLONG_MAX / 1000". * Document the change and give hints how to fix affected code. * Add an exception for PY_TIMEOUT_MAX name to smelly.py * Add PY_TIMEOUT_MAX to the stable ABI files: M Doc/data/stable_abi.dat M Doc/whatsnew/3.13.rst M Include/internal/pycore_condvar.h M Include/internal/pycore_pythread.h M Include/internal/pycore_semaphore.h M Include/pythread.h M Lib/test/test_stable_abi_ctypes.py M Misc/stable_abi.toml M PC/python3dll.c M Python/condvar.h M Python/thread.c M Python/thread_pthread.h M Tools/build/smelly.py diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index c189c78238f40..07c6d514d1954 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,4 +1,5 @@ role,name,added,ifdef_note,struct_abi_kind +var,PY_TIMEOUT_MAX,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, function,PyAIter_Check,3.10,, function,PyArg_Parse,3.2,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c9e6ca8bf8886..d6188e63dd23e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -989,6 +989,10 @@ Porting to Python 3.13 * ``Python.h`` no longer includes the ``<unistd.h>`` standard header file. If needed, it should now be included explicitly. For example, it provides the functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. + As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no + longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H`` + macros defined by ``Python.h`` can be used to decide if ``<unistd.h>`` and + ``<pthread.h>`` header files can be included. (Contributed by Victor Stinner in :gh:`108765`.) * ``Python.h`` no longer includes these standard header files: ``<time.h>``, diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 489e67d4ec4f9..34c21aaad4319 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,18 +5,8 @@ # error "this header requires Py_BUILD_CORE define" #endif -#ifndef MS_WINDOWS -# include <unistd.h> // _POSIX_THREADS -#endif +#include "pycore_pythread.h" // _POSIX_THREADS -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include <pthread.h> // _POSIX_THREADS -# endif -#endif #ifdef _POSIX_THREADS /* diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 5ec2abda91e86..98019c586bc5e 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -9,29 +9,29 @@ extern "C" { #endif -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include <pthread.h> // _POSIX_THREADS -# endif -# ifndef _POSIX_THREADS -/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then - enough of the Posix threads package is implemented to support python - threads. - - This is valid for HP-UX 11.23 running on an ia64 system. If needed, add - a check of __ia64 to verify that we're running on an ia64 system instead - of a pa-risc system. -*/ -# ifdef __hpux -# ifdef _SC_THREADS -# define _POSIX_THREADS -# endif -# endif -# endif /* _POSIX_THREADS */ -#endif /* _POSIX_THREADS */ +// Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available +#if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) +# include <unistd.h> // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if (defined(HAVE_PTHREAD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) + // This means pthreads are not implemented in libc headers, hence the macro + // not present in <unistd.h>. But they still can be implemented as an + // external library (e.g. gnu pth in pthread emulation) +# include <pthread.h> // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if !defined(_POSIX_THREADS) && defined(__hpux) && defined(_SC_THREADS) + // Check if we're running on HP-UX and _SC_THREADS is defined. If so, then + // enough of the POSIX threads package is implemented to support Python + // threads. + // + // This is valid for HP-UX 11.23 running on an ia64 system. If needed, add + // a check of __ia64 to verify that we're running on an ia64 system instead + // of a pa-risc system. +# define _POSIX_THREADS +#endif + #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) # define _USE_PTHREADS diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 2a4ecb7147ace..4c37df7b39a48 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -7,7 +7,8 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t +#include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN @@ -26,6 +27,7 @@ # include <semaphore.h> #endif + #ifdef __cplusplus extern "C" { #endif diff --git a/Include/pythread.h b/Include/pythread.h index 63714437c496b..2c2fd63d72428 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -44,22 +44,7 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); */ #define PY_TIMEOUT_T long long -#if defined(_POSIX_THREADS) - /* PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), - convert microseconds to nanoseconds. */ -# define PY_TIMEOUT_MAX (LLONG_MAX / 1000) -#elif defined (NT_THREADS) - // WaitForSingleObject() accepts timeout in milliseconds in the range - // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no - // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. -# if 0xFFFFFFFELL * 1000 < LLONG_MAX -# define PY_TIMEOUT_MAX (0xFFFFFFFELL * 1000) -# else -# define PY_TIMEOUT_MAX LLONG_MAX -# endif -#else -# define PY_TIMEOUT_MAX LLONG_MAX -#endif +PyAPI_DATA(const long long) PY_TIMEOUT_MAX; /* If microseconds == 0, the call is non-blocking: it returns immediately diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 94f817f8e1d15..6e9496d40da47 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -35,6 +35,7 @@ def test_windows_feature_macros(self): SYMBOL_NAMES = ( + "PY_TIMEOUT_MAX", "PyAIter_Check", "PyArg_Parse", "PyArg_ParseTuple", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8df3f85e61eec..46e2307614e26 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1843,6 +1843,10 @@ [function.PyThread_start_new_thread] added = '3.2' +# Not mentioned in PEP 384, was implemented as a macro in Python <= 3.12 +[data.PY_TIMEOUT_MAX] + added = '3.2' + # The following were added in PC/python3.def in Python 3.3: # 7800f75827b1be557be16f3b18f5170fbf9fae08 # 9c56409d3353b8cd4cfc19e0467bbe23fd34fc92 diff --git a/PC/python3dll.c b/PC/python3dll.c index 2c1cc8098ce85..75728c7d8057e 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -768,6 +768,7 @@ EXPORT_DATA(Py_FileSystemDefaultEncodeErrors) EXPORT_DATA(Py_FileSystemDefaultEncoding) EXPORT_DATA(Py_GenericAliasType) EXPORT_DATA(Py_HasFileSystemDefaultEncoding) +EXPORT_DATA(PY_TIMEOUT_MAX) EXPORT_DATA(Py_UTF8Mode) EXPORT_DATA(Py_Version) EXPORT_DATA(PyBaseObject_Type) diff --git a/Python/condvar.h b/Python/condvar.h index 4ddc5311cf8fa..d54db94f2c871 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -41,7 +41,8 @@ #define _CONDVAR_IMPL_H_ #include "Python.h" -#include "pycore_condvar.h" +#include "pycore_pythread.h" // _POSIX_THREADS + #ifdef _POSIX_THREADS /* diff --git a/Python/thread.c b/Python/thread.c index 1ac2db2937e37..bf207cecb9050 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -8,7 +8,7 @@ #include "Python.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_pythread.h" +#include "pycore_pythread.h" // _POSIX_THREADS #ifndef DONT_HAVE_STDIO_H # include <stdio.h> @@ -17,6 +17,26 @@ #include <stdlib.h> +// Define PY_TIMEOUT_MAX constant. +#ifdef _POSIX_THREADS + // PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), + // convert microseconds to nanoseconds. +# define PY_TIMEOUT_MAX_VALUE (LLONG_MAX / 1000) +#elif defined (NT_THREADS) + // WaitForSingleObject() accepts timeout in milliseconds in the range + // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no + // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. +# if 0xFFFFFFFELL < LLONG_MAX / 1000 +# define PY_TIMEOUT_MAX_VALUE (0xFFFFFFFELL * 1000) +# else +# define PY_TIMEOUT_MAX_VALUE LLONG_MAX +# endif +#else +# define PY_TIMEOUT_MAX_VALUE LLONG_MAX +#endif +const long long PY_TIMEOUT_MAX = PY_TIMEOUT_MAX_VALUE; + + static void PyThread__init_thread(void); /* Forward */ #define initialized _PyRuntime.threads.initialized diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index f96c57da64636..76a1f7763f23b 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,4 +1,5 @@ -#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_pythread.h" // _POSIX_SEMAPHORES /* Posix threads interface */ @@ -84,10 +85,10 @@ /* On FreeBSD 4.x, _POSIX_SEMAPHORES is defined empty, so we need to add 0 to make it work there as well. */ #if (_POSIX_SEMAPHORES+0) == -1 -#define HAVE_BROKEN_POSIX_SEMAPHORES +# define HAVE_BROKEN_POSIX_SEMAPHORES #else -#include <semaphore.h> -#include <errno.h> +# include <semaphore.h> +# include <errno.h> #endif #endif diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 276a5ab2cc84c..ab345307ff9b6 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -11,6 +11,11 @@ if sys.platform == 'darwin': ALLOWED_PREFIXES += ('__Py',) +# "Legacy": some old symbols are prefixed by "PY_". +EXCEPTIONS = frozenset({ + 'PY_TIMEOUT_MAX', +}) + IGNORED_EXTENSION = "_ctypes_test" # Ignore constructor and destructor functions IGNORED_SYMBOLS = {'_init', '_fini'} @@ -72,7 +77,7 @@ def get_smelly_symbols(stdout): symbol = parts[-1] result = '%s (type: %s)' % (symbol, symtype) - if symbol.startswith(ALLOWED_PREFIXES): + if symbol.startswith(ALLOWED_PREFIXES) or symbol in EXCEPTIONS: python_symbols.append(result) continue

1 0