Mailman 3 April 2022 - Python-checkins

bpo-47000: Make `io.text_encoding()` respects UTF-8 mode (GH-32003)
by methane April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/4216dce04b7d3f329beaaafc82a77c4ac6… commit: 4216dce04b7d3f329beaaafc82a77c4ac6cf4d57 branch: main author: Inada Naoki <songofacandy(a)gmail.com> committer: methane <songofacandy(a)gmail.com> date: 2022-04-04T11:46:57+09:00 summary: bpo-47000: Make `io.text_encoding()` respects UTF-8 mode (GH-32003) Co-authored-by: Eric Snow <ericsnowcurrently(a)gmail.com> files: A Misc/NEWS.d/next/Library/2022-03-20-13-00-08.bpo-47000.p8HpG0.rst M Doc/library/io.rst M Include/internal/pycore_global_strings.h M Include/internal/pycore_runtime_init.h M Lib/_pyio.py M Lib/test/test_io.py M Lib/test/test_utf8_mode.py M Modules/_io/_iomodule.c M Modules/_io/clinic/_iomodule.c.h M Python/sysmodule.c diff --git a/Doc/library/io.rst b/Doc/library/io.rst index d5123348195bd..80107d539505c 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -198,12 +198,13 @@ High-level Module Interface This is a helper function for callables that use :func:`open` or :class:`TextIOWrapper` and have an ``encoding=None`` parameter. - This function returns *encoding* if it is not ``None`` and ``"locale"`` if - *encoding* is ``None``. + This function returns *encoding* if it is not ``None``. + Otherwise, it returns ``"locale"`` or ``"utf-8"`` depending on + :ref:`UTF-8 Mode <utf8-mode>`. This function emits an :class:`EncodingWarning` if :data:`sys.flags.warn_default_encoding <sys.flags>` is true and *encoding* - is None. *stacklevel* specifies where the warning is emitted. + is ``None``. *stacklevel* specifies where the warning is emitted. For example:: def read_text(path, encoding=None): @@ -218,6 +219,10 @@ High-level Module Interface .. versionadded:: 3.10 + .. versionchanged:: 3.11 + :func:`text_encoding` returns "utf-8" when UTF-8 mode is enabled and + *encoding* is ``None``. + .. exception:: BlockingIOError diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3e533fd16509f..833ff2710a787 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -48,6 +48,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(newline, "\n") STRUCT_FOR_STR(open_br, "{") STRUCT_FOR_STR(percent, "%") + STRUCT_FOR_STR(utf_8, "utf-8") } literals; struct { diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index d5690d83a0482..fd925b3e060df 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -672,6 +672,7 @@ extern "C" { INIT_STR(newline, "\n"), \ INIT_STR(open_br, "{"), \ INIT_STR(percent, "%"), \ + INIT_STR(utf_8, "utf-8"), \ }, \ .identifiers = { \ INIT_ID(False), \ diff --git a/Lib/_pyio.py b/Lib/_pyio.py index fd00d6536c076..e3ff59eb1adb1 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -44,8 +44,9 @@ def text_encoding(encoding, stacklevel=2): """ A helper function to choose the text encoding. - When encoding is not None, just return it. - Otherwise, return the default text encoding (i.e. "locale"). + When encoding is not None, this function returns it. + Otherwise, this function returns the default text encoding + (i.e. "locale" or "utf-8" depends on UTF-8 mode). This function emits an EncodingWarning if *encoding* is None and sys.flags.warn_default_encoding is true. @@ -55,7 +56,10 @@ def text_encoding(encoding, stacklevel=2): However, please consider using encoding="utf-8" for new APIs. """ if encoding is None: - encoding = "locale" + if sys.flags.utf8_mode: + encoding = "utf-8" + else: + encoding = "locale" if sys.flags.warn_default_encoding: import warnings warnings.warn("'encoding' argument not specified.", diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 2d0ca878788f2..67be108d2526f 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4289,6 +4289,17 @@ def test_check_encoding_warning(self): self.assertTrue( warnings[1].startswith(b"<string>:8: EncodingWarning: ")) + def test_text_encoding(self): + # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" + # based on sys.flags.utf8_mode + code = "import io; print(io.text_encoding(None))" + + proc = assert_python_ok('-X', 'utf8=0', '-c', code) + self.assertEqual(b"locale", proc.out.strip()) + + proc = assert_python_ok('-X', 'utf8=1', '-c', code) + self.assertEqual(b"utf-8", proc.out.strip()) + @support.cpython_only # Depending if OpenWrapper was already created or not, the warning is # emitted or not. For example, the attribute is already created when this diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index 2b96f76df305f..308e8e8aea6c2 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -161,7 +161,7 @@ def test_io(self): filename = __file__ out = self.get_output('-c', code, filename, PYTHONUTF8='1') - self.assertEqual(out, 'UTF-8/strict') + self.assertEqual(out.lower(), 'utf-8/strict') def _check_io_encoding(self, module, encoding=None, errors=None): filename = __file__ @@ -183,10 +183,10 @@ def _check_io_encoding(self, module, encoding=None, errors=None): PYTHONUTF8='1') if not encoding: - encoding = 'UTF-8' + encoding = 'utf-8' if not errors: errors = 'strict' - self.assertEqual(out, f'{encoding}/{errors}') + self.assertEqual(out.lower(), f'{encoding}/{errors}') def check_io_encoding(self, module): self._check_io_encoding(module, encoding="latin1") diff --git a/Misc/NEWS.d/next/Library/2022-03-20-13-00-08.bpo-47000.p8HpG0.rst b/Misc/NEWS.d/next/Library/2022-03-20-13-00-08.bpo-47000.p8HpG0.rst new file mode 100644 index 0000000000000..f96b6e627ed11 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-20-13-00-08.bpo-47000.p8HpG0.rst @@ -0,0 +1 @@ +Make :func:`io.text_encoding` returns "utf-8" when UTF-8 mode is enabled. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 7f029f26078b8..065f5e29c315b 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -457,8 +457,9 @@ _io.text_encoding A helper function to choose the text encoding. -When encoding is not None, just return it. -Otherwise, return the default text encoding (i.e. "locale"). +When encoding is not None, this function returns it. +Otherwise, this function returns the default text encoding +(i.e. "locale" or "utf-8" depends on UTF-8 mode). This function emits an EncodingWarning if encoding is None and sys.flags.warn_default_encoding is true. @@ -469,7 +470,7 @@ However, please consider using encoding="utf-8" for new APIs. static PyObject * _io_text_encoding_impl(PyObject *module, PyObject *encoding, int stacklevel) -/*[clinic end generated code: output=91b2cfea6934cc0c input=bf70231213e2a7b4]*/ +/*[clinic end generated code: output=91b2cfea6934cc0c input=4999aa8b3d90f3d4]*/ { if (encoding == NULL || encoding == Py_None) { PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -479,7 +480,14 @@ _io_text_encoding_impl(PyObject *module, PyObject *encoding, int stacklevel) return NULL; } } - return &_Py_ID(locale); + const PyPreConfig *preconfig = &_PyRuntime.preconfig; + if (preconfig->utf8_mode) { + _Py_DECLARE_STR(utf_8, "utf-8"); + encoding = &_Py_STR(utf_8); + } + else { + encoding = &_Py_ID(locale); + } } Py_INCREF(encoding); return encoding; diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index d5fb176eb66be..e4a6b8c42e1d8 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -273,8 +273,9 @@ PyDoc_STRVAR(_io_text_encoding__doc__, "\n" "A helper function to choose the text encoding.\n" "\n" -"When encoding is not None, just return it.\n" -"Otherwise, return the default text encoding (i.e. \"locale\").\n" +"When encoding is not None, this function returns it.\n" +"Otherwise, this function returns the default text encoding\n" +"(i.e. \"locale\" or \"utf-8\" depends on UTF-8 mode).\n" "\n" "This function emits an EncodingWarning if encoding is None and\n" "sys.flags.warn_default_encoding is true.\n" @@ -354,4 +355,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=6ea315343f6a94ba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1a7fd7755c9a9609 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5765e9ef6577c..de4e10a7e110c 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -841,7 +841,10 @@ static PyObject * sys_getdefaultencoding_impl(PyObject *module) /*[clinic end generated code: output=256d19dfcc0711e6 input=d416856ddbef6909]*/ { - return PyUnicode_FromString(PyUnicode_GetDefaultEncoding()); + _Py_DECLARE_STR(utf_8, "utf-8"); + PyObject *ret = &_Py_STR(utf_8); + Py_INCREF(ret); + return ret; } /*[clinic input]

1 0

Follow PEP-8 guidelines in tutorial for standard library (GH-26127)
by miss-islington April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/0d0a6f18a09d04244af3a09b50b65c3d0c… commit: 0d0a6f18a09d04244af3a09b50b65c3d0ca43fd2 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-03T15:58:21-07:00 summary: Follow PEP-8 guidelines in tutorial for standard library (GH-26127) (cherry picked from commit 6db2db91b96aaa1270c200ec931a2250fe2799c7) Co-authored-by: Bob Kline <bkline(a)users.noreply.github.com> files: M Doc/tutorial/stdlib.rst diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index ab64ca6d400b8..a6399dc1af3a9 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -78,8 +78,9 @@ and an optional number of lines to be displayed:: import argparse - parser = argparse.ArgumentParser(prog = 'top', - description = 'Show top lines from each file') + parser = argparse.ArgumentParser( + prog='top', + description='Show top lines from each file') parser.add_argument('filenames', nargs='+') parser.add_argument('-l', '--lines', type=int, default=10) args = parser.parse_args()

1 0

Follow PEP-8 guidelines in tutorial for standard library (GH-26127)
by miss-islington April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/a331d0f627406d621fa8ef1a1407f2c8a4… commit: a331d0f627406d621fa8ef1a1407f2c8a4cee4b5 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-03T15:49:52-07:00 summary: Follow PEP-8 guidelines in tutorial for standard library (GH-26127) (cherry picked from commit 6db2db91b96aaa1270c200ec931a2250fe2799c7) Co-authored-by: Bob Kline <bkline(a)users.noreply.github.com> files: M Doc/tutorial/stdlib.rst diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index ac16160b23439..aac1ae3a8a6b4 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -78,8 +78,9 @@ and an optional number of lines to be displayed:: import argparse - parser = argparse.ArgumentParser(prog = 'top', - description = 'Show top lines from each file') + parser = argparse.ArgumentParser( + prog='top', + description='Show top lines from each file') parser.add_argument('filenames', nargs='+') parser.add_argument('-l', '--lines', type=int, default=10) args = parser.parse_args()

1 0

Follow PEP-8 guidelines in tutorial for standard library (GH-26127)
by JelleZijlstra April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/6db2db91b96aaa1270c200ec931a2250fe… commit: 6db2db91b96aaa1270c200ec931a2250fe2799c7 branch: main author: Bob Kline <bkline(a)users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2022-04-03T15:31:03-07:00 summary: Follow PEP-8 guidelines in tutorial for standard library (GH-26127) files: M Doc/tutorial/stdlib.rst diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index d90dc51c71927..227a6d8651ff5 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -78,8 +78,9 @@ and an optional number of lines to be displayed:: import argparse - parser = argparse.ArgumentParser(prog = 'top', - description = 'Show top lines from each file') + parser = argparse.ArgumentParser( + prog='top', + description='Show top lines from each file') parser.add_argument('filenames', nargs='+') parser.add_argument('-l', '--lines', type=int, default=10) args = parser.parse_args()

1 0

bpo-47176: Interrupt handling for wasm32-emscripten builds without pthreads (GH-32209)
by tiran April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/087d0fa5b97796560c0d8ceab4f0360fd5… commit: 087d0fa5b97796560c0d8ceab4f0360fd54baf4f branch: main author: Hood Chatham <roberthoodchatham(a)gmail.com> committer: tiran <christian(a)python.org> date: 2022-04-03T22:58:52+02:00 summary: bpo-47176: Interrupt handling for wasm32-emscripten builds without pthreads (GH-32209) Co-authored-by: Christian Heimes <christian(a)python.org> Co-authored-by: Brett Cannon <brett(a)python.org> files: A Include/internal/pycore_emscripten_signal.h A Misc/NEWS.d/next/Core and Builtins/2022-04-02-14-32-21.bpo-47176.kTygYI.rst A Python/emscripten_signal.c M Makefile.pre.in M Modules/signalmodule.c M Python/ceval.c M configure M configure.ac diff --git a/Include/internal/pycore_emscripten_signal.h b/Include/internal/pycore_emscripten_signal.h new file mode 100644 index 0000000000000..8b3287d85da4b --- /dev/null +++ b/Include/internal/pycore_emscripten_signal.h @@ -0,0 +1,25 @@ +#ifndef Py_EMSCRIPTEN_SIGNAL_H +#define Py_EMSCRIPTEN_SIGNAL_H + +#if defined(__EMSCRIPTEN__) + +void +_Py_CheckEmscriptenSignals(void); + +void +_Py_CheckEmscriptenSignalsPeriodically(void); + +#define _Py_CHECK_EMSCRIPTEN_SIGNALS() _Py_CheckEmscriptenSignals() + +#define _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY() _Py_CheckEmscriptenSignalsPeriodically() + +extern int Py_EMSCRIPTEN_SIGNAL_HANDLING; + +#else + +#define _Py_CHECK_EMSCRIPTEN_SIGNALS() +#define _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY() + +#endif // defined(__EMSCRIPTEN__) + +#endif // ndef Py_EMSCRIPTEN_SIGNAL_H diff --git a/Makefile.pre.in b/Makefile.pre.in index 5318a41dc857a..f94ba93cff91c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -429,7 +429,8 @@ PYTHON_OBJS= \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ - $(DTRACE_OBJS) + $(DTRACE_OBJS) \ + @PLATFORM_OBJS@ ########################################################################## @@ -1608,6 +1609,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_unicodeobject.h \ $(srcdir)/Include/internal/pycore_warnings.h \ $(DTRACE_HEADERS) \ + @PLATFORM_HEADERS@ \ \ $(srcdir)/Python/stdlib_module_names.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-04-02-14-32-21.bpo-47176.kTygYI.rst b/Misc/NEWS.d/next/Core and Builtins/2022-04-02-14-32-21.bpo-47176.kTygYI.rst new file mode 100644 index 0000000000000..03fe54a372552 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-04-02-14-32-21.bpo-47176.kTygYI.rst @@ -0,0 +1,6 @@ +Emscripten builds cannot handle signals in the usual way due to platform +limitations. Python can now handle signals. To use, set +Module.Py_EmscriptenSignalBuffer to be a single byte SharedArrayBuffer and +set Py_EMSCRIPTEN_SIGNAL_HANDLING to 1. Writing a number into the +SharedArrayBuffer will cause the corresponding signal to be raised into the +Python thread. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 9566263a0dd87..1ee5c669df015 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,6 +13,7 @@ #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pylifecycle.h" // NSIG #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #ifndef MS_WINDOWS # include "posixmodule.h" @@ -1797,6 +1798,7 @@ PyErr_CheckSignals(void) int _PyErr_CheckSignalsTstate(PyThreadState *tstate) { + _Py_CHECK_EMSCRIPTEN_SIGNALS(); if (!_Py_atomic_load(&is_tripped)) { return 0; } diff --git a/Python/ceval.c b/Python/ceval.c index 43080f8db0422..68d2920727ab0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -21,6 +21,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "code.h" #include "pycore_dict.h" @@ -1292,6 +1293,7 @@ eval_frame_handle_pending(PyThreadState *tstate) } #define CHECK_EVAL_BREAKER() \ + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ if (_Py_atomic_load_relaxed(eval_breaker)) { \ goto handle_eval_breaker; \ } diff --git a/Python/emscripten_signal.c b/Python/emscripten_signal.c new file mode 100644 index 0000000000000..d617ddfeb37c5 --- /dev/null +++ b/Python/emscripten_signal.c @@ -0,0 +1,56 @@ +// To enable signal handling, the embedder should: +// 1. set Module.Py_EmscriptenSignalBuffer = some_shared_array_buffer; +// 2. set the Py_EMSCRIPTEN_SIGNAL_HANDLING flag to 1 as follows: +// Module.HEAP8[Module._Py_EMSCRIPTEN_SIGNAL_HANDLING] = 1 +// +// The address &Py_EMSCRIPTEN_SIGNAL_HANDLING is exported as +// Module._Py_EMSCRIPTEN_SIGNAL_HANDLING. +#include <emscripten.h> +#include "Python.h" + +EM_JS(int, _Py_CheckEmscriptenSignals_Helper, (void), { + if (!Module.Py_EmscriptenSignalBuffer) { + return 0; + } + try { + let result = Module.Py_EmscriptenSignalBuffer[0]; + Module.Py_EmscriptenSignalBuffer[0] = 0; + return result; + } catch(e) { +#if !defined(NDEBUG) + console.warn("Error occurred while trying to read signal buffer:", e); +#endif + return 0; + } +}); + +EMSCRIPTEN_KEEPALIVE int Py_EMSCRIPTEN_SIGNAL_HANDLING = 0; + +void +_Py_CheckEmscriptenSignals(void) +{ + if (!Py_EMSCRIPTEN_SIGNAL_HANDLING) { + return; + } + int signal = _Py_CheckEmscriptenSignals_Helper(); + if (signal) { + PyErr_SetInterruptEx(signal); + } +} + + +#define PY_EMSCRIPTEN_SIGNAL_INTERVAL 50 +static int emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; + +void +_Py_CheckEmscriptenSignalsPeriodically(void) +{ + if (!Py_EMSCRIPTEN_SIGNAL_HANDLING) { + return; + } + emscripten_signal_clock--; + if (emscripten_signal_clock == 0) { + emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; + _Py_CheckEmscriptenSignals(); + } +} diff --git a/configure b/configure index a5062d7b81573..bb1aa7568233d 100755 --- a/configure +++ b/configure @@ -817,6 +817,8 @@ TRUE MACHDEP_OBJS DYNLOADFILE DLINCLDIR +PLATFORM_OBJS +PLATFORM_HEADERS DTRACE_OBJS DTRACE_HEADERS DFLAGS @@ -13993,6 +13995,21 @@ $as_echo "$ac_cv_dtrace_link" >&6; } fi fi +PLATFORM_HEADERS= +PLATFORM_OBJS= + +case $ac_sys_system in #( + Emscripten) : + + as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o' + as_fn_append PLATFORM_HEADERS ' $(srcdir)/Include/internal/pycore_emscripten_signal.h' + ;; #( + *) : + ;; +esac + + + # -I${DLINCLDIR} is added to the compile rule for importdl.o DLINCLDIR=. diff --git a/configure.ac b/configure.ac index 84bc9b3ca5cf0..9f0a50ec852b6 100644 --- a/configure.ac +++ b/configure.ac @@ -4187,6 +4187,19 @@ then fi fi +dnl Platform-specific C and header files. +PLATFORM_HEADERS= +PLATFORM_OBJS= + +AS_CASE([$ac_sys_system], + [Emscripten], [ + AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o']) + AS_VAR_APPEND([PLATFORM_HEADERS], [' $(srcdir)/Include/internal/pycore_emscripten_signal.h']) + ], +) +AC_SUBST([PLATFORM_HEADERS]) +AC_SUBST([PLATFORM_OBJS]) + # -I${DLINCLDIR} is added to the compile rule for importdl.o AC_SUBST(DLINCLDIR) DLINCLDIR=.

1 0

bpo-42238: [doc]: make suspicious: false positive. (GH-32292)
by JulienPalard April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/bdc497496548e30fa208a8d98c30bf6d18… commit: bdc497496548e30fa208a8d98c30bf6d1833ac4c branch: main author: Julien Palard <julien(a)palard.fr> committer: JulienPalard <julien(a)palard.fr> date: 2022-04-03T22:08:29+02:00 summary: bpo-42238: [doc]: make suspicious: false positive. (GH-32292) files: M Doc/tools/susp-ignored.csv diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index a5ceff4be0196..88f401fc8370a 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -392,3 +392,4 @@ whatsnew/changelog,,:CON,": os.path.abspath(“C:CON”) is now fixed to return whatsnew/changelog,,::,Lib/email/mime/nonmultipart.py::MIMENonMultipart library/typing,,`,"assert_type(name, str) # OK, inferred type of `name` is `str`" library/typing,,`,# after which we hope the inferred type will be `int` +whatsnew/changelog,,:company,-V:company/tag

1 0

bpo-46126: Restore 'descriptions' when running tests internally. (GH-32128)
by miss-islington April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/84acb5cad1b871bb8729cbf1d036b84cbe… commit: 84acb5cad1b871bb8729cbf1d036b84cbe1636f0 branch: main author: Jason R. Coombs <jaraco(a)jaraco.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-03T12:33:28-07:00 summary: bpo-46126: Restore 'descriptions' when running tests internally. (GH-32128) This reverts commit a941e5927f7f2540946813606c61c6aea38db426 (GH-30194). Automerge-Triggered-By: GH:jaraco files: A Misc/NEWS.d/next/Tests/2022-03-26-11-41-19.bpo-46126.q14Ioy.rst M Lib/test/support/testresult.py diff --git a/Lib/test/support/testresult.py b/Lib/test/support/testresult.py index eb2279a88f9a0..2cd1366cd8a9e 100644 --- a/Lib/test/support/testresult.py +++ b/Lib/test/support/testresult.py @@ -145,11 +145,7 @@ def get_test_runner_class(verbosity, buffer=False): return functools.partial(unittest.TextTestRunner, resultclass=RegressionTestResult, buffer=buffer, - verbosity=verbosity, - # disable descriptions so errors are - # readily traceable. bpo-46126 - descriptions=False, - ) + verbosity=verbosity) return functools.partial(QuietRegressionTestRunner, buffer=buffer) def get_test_runner(stream, verbosity, capture_output=False): diff --git a/Misc/NEWS.d/next/Tests/2022-03-26-11-41-19.bpo-46126.q14Ioy.rst b/Misc/NEWS.d/next/Tests/2022-03-26-11-41-19.bpo-46126.q14Ioy.rst new file mode 100644 index 0000000000000..0877b0e385743 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-03-26-11-41-19.bpo-46126.q14Ioy.rst @@ -0,0 +1 @@ +Restore 'descriptions' when running tests internally.

1 0

bpo-47205: Skip error check of sched_get/setaffinity on FreeBSD (GH-32285)
by miss-islington April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/470dfe20cb6e741c42c52619e122fc218e… commit: 470dfe20cb6e741c42c52619e122fc218e27aebd branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-03T12:27:32-07:00 summary: bpo-47205: Skip error check of sched_get/setaffinity on FreeBSD (GH-32285) (cherry picked from commit b82cdd1dac9a9be52051abd90a1ce69236ac41f4) Co-authored-by: Christian Heimes <christian(a)python.org> files: A Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst M Lib/test/test_posix.py diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 974edd766cc80..701543bb6ac66 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1179,7 +1179,9 @@ def test_sched_getaffinity(self): mask = posix.sched_getaffinity(0) self.assertIsInstance(mask, set) self.assertGreaterEqual(len(mask), 1) - self.assertRaises(OSError, posix.sched_getaffinity, -1) + if not sys.platform.startswith("freebsd"): + # bpo-47205: does not raise OSError on FreeBSD + self.assertRaises(OSError, posix.sched_getaffinity, -1) for cpu in mask: self.assertIsInstance(cpu, int) self.assertGreaterEqual(cpu, 0) @@ -1197,7 +1199,9 @@ def test_sched_setaffinity(self): self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) - self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) + if not sys.platform.startswith("freebsd"): + # bpo-47205: does not raise OSError on FreeBSD + self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) def test_rtld_constants(self): # check presence of major RTLD_* constants diff --git a/Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst b/Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst new file mode 100644 index 0000000000000..35fd94421326e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst @@ -0,0 +1,2 @@ +Skip test for :func:`~os.sched_getaffinity` and +:func:`~os.sched_setaffinity` error case on FreeBSD.

1 0

bpo-23689: re module, fix memory leak when a match is terminated by a signal or memory allocation failure (GH-32283)
by serhiy-storchaka April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/6e3eee5c11b539e9aab39cff783acf5783… commit: 6e3eee5c11b539e9aab39cff783acf57838c355a branch: main author: Ma Lin <animalize(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2022-04-03T19:16:20+03:00 summary: bpo-23689: re module, fix memory leak when a match is terminated by a signal or memory allocation failure (GH-32283) files: A Misc/NEWS.d/next/Library/2022-04-03-13-19-08.bpo-23689.TFSc3E.rst M Lib/re/_compiler.py M Lib/re/_constants.py M Lib/test/test_re.py M Modules/_sre.c M Modules/clinic/_sre.c.h M Modules/sre.h M Modules/sre_constants.h M Modules/sre_lib.h diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 62da8e55d72ab..bedd4b8f40016 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -67,14 +67,21 @@ _ignorecase_fixes = {i: tuple(j for j in t if i != j) for t in _equivalences for i in t} +class _CompileData: + __slots__ = ('code', 'repeat_count') + def __init__(self): + self.code = [] + self.repeat_count = 0 + def _combine_flags(flags, add_flags, del_flags, TYPE_FLAGS=_parser.TYPE_FLAGS): if add_flags & TYPE_FLAGS: flags &= ~TYPE_FLAGS return (flags | add_flags) & ~del_flags -def _compile(code, pattern, flags): +def _compile(data, pattern, flags): # internal: compile a (sub)pattern + code = data.code emit = code.append _len = len LITERAL_CODES = _LITERAL_CODES @@ -147,7 +154,7 @@ def _compile(code, pattern, flags): skip = _len(code); emit(0) emit(av[0]) emit(av[1]) - _compile(code, av[2], flags) + _compile(data, av[2], flags) emit(SUCCESS) code[skip] = _len(code) - skip else: @@ -155,7 +162,11 @@ def _compile(code, pattern, flags): skip = _len(code); emit(0) emit(av[0]) emit(av[1]) - _compile(code, av[2], flags) + # now op is in (MIN_REPEAT, MAX_REPEAT, POSSESSIVE_REPEAT) + if op != POSSESSIVE_REPEAT: + emit(data.repeat_count) + data.repeat_count += 1 + _compile(data, av[2], flags) code[skip] = _len(code) - skip emit(REPEATING_CODES[op][1]) elif op is SUBPATTERN: @@ -164,7 +175,7 @@ def _compile(code, pattern, flags): emit(MARK) emit((group-1)*2) # _compile_info(code, p, _combine_flags(flags, add_flags, del_flags)) - _compile(code, p, _combine_flags(flags, add_flags, del_flags)) + _compile(data, p, _combine_flags(flags, add_flags, del_flags)) if group: emit(MARK) emit((group-1)*2+1) @@ -176,7 +187,7 @@ def _compile(code, pattern, flags): # pop their stack if they reach it emit(ATOMIC_GROUP) skip = _len(code); emit(0) - _compile(code, av, flags) + _compile(data, av, flags) emit(SUCCESS) code[skip] = _len(code) - skip elif op in SUCCESS_CODES: @@ -191,13 +202,13 @@ def _compile(code, pattern, flags): if lo != hi: raise error("look-behind requires fixed-width pattern") emit(lo) # look behind - _compile(code, av[1], flags) + _compile(data, av[1], flags) emit(SUCCESS) code[skip] = _len(code) - skip elif op is CALL: emit(op) skip = _len(code); emit(0) - _compile(code, av, flags) + _compile(data, av, flags) emit(SUCCESS) code[skip] = _len(code) - skip elif op is AT: @@ -216,7 +227,7 @@ def _compile(code, pattern, flags): for av in av[1]: skip = _len(code); emit(0) # _compile_info(code, av, flags) - _compile(code, av, flags) + _compile(data, av, flags) emit(JUMP) tailappend(_len(code)); emit(0) code[skip] = _len(code) - skip @@ -244,12 +255,12 @@ def _compile(code, pattern, flags): emit(op) emit(av[0]-1) skipyes = _len(code); emit(0) - _compile(code, av[1], flags) + _compile(data, av[1], flags) if av[2]: emit(JUMP) skipno = _len(code); emit(0) code[skipyes] = _len(code) - skipyes + 1 - _compile(code, av[2], flags) + _compile(data, av[2], flags) code[skipno] = _len(code) - skipno else: code[skipyes] = _len(code) - skipyes + 1 @@ -608,17 +619,17 @@ def isstring(obj): def _code(p, flags): flags = p.state.flags | flags - code = [] + data = _CompileData() # compile info block - _compile_info(code, p, flags) + _compile_info(data.code, p, flags) # compile the pattern - _compile(code, p.data, flags) + _compile(data, p.data, flags) - code.append(SUCCESS) + data.code.append(SUCCESS) - return code + return data def _hex_code(code): return '[%s]' % ', '.join('%#0*x' % (_sre.CODESIZE*2+2, x) for x in code) @@ -719,7 +730,7 @@ def print_2(*args): else: print_(FAILURE) i += 1 - elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE, + elif op in (REPEAT_ONE, MIN_REPEAT_ONE, POSSESSIVE_REPEAT, POSSESSIVE_REPEAT_ONE): skip, min, max = code[i: i+3] if max == MAXREPEAT: @@ -727,6 +738,13 @@ def print_2(*args): print_(op, skip, min, max, to=i+skip) dis_(i+3, i+skip) i += skip + elif op is REPEAT: + skip, min, max, repeat_index = code[i: i+4] + if max == MAXREPEAT: + max = 'MAXREPEAT' + print_(op, skip, min, max, repeat_index, to=i+skip) + dis_(i+4, i+skip) + i += skip elif op is GROUPREF_EXISTS: arg, skip = code[i: i+2] print_(op, arg, skip, to=i+skip) @@ -781,11 +799,11 @@ def compile(p, flags=0): else: pattern = None - code = _code(p, flags) + data = _code(p, flags) if flags & SRE_FLAG_DEBUG: print() - dis(code) + dis(data.code) # map in either direction groupindex = p.state.groupdict @@ -794,7 +812,6 @@ def compile(p, flags=0): indexgroup[i] = k return _sre.compile( - pattern, flags | p.state.flags, code, - p.state.groups-1, - groupindex, tuple(indexgroup) - ) + pattern, flags | p.state.flags, data.code, + p.state.groups-1, groupindex, tuple(indexgroup), + data.repeat_count) diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index c735edfea1f13..5317fd53e9c5a 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -13,7 +13,7 @@ # update when constants are added or removed -MAGIC = 20220318 +MAGIC = 20220402 from _sre import MAXREPEAT, MAXGROUPS diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index f1e5af452d8e0..553eb4cfe85b6 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1643,9 +1643,12 @@ def test_dealloc(self): long_overflow = 2**128 self.assertRaises(TypeError, re.finditer, "a", {}) with self.assertRaises(OverflowError): - _sre.compile("abc", 0, [long_overflow], 0, {}, ()) + _sre.compile("abc", 0, [long_overflow], 0, {}, (), 0) with self.assertRaises(TypeError): - _sre.compile({}, 0, [], 0, [], []) + _sre.compile({}, 0, [], 0, [], [], 0) + with self.assertRaises(RuntimeError): + # invalid repeat_count -1 + _sre.compile("abc", 0, [1], 0, {}, (), -1) def test_search_dot_unicode(self): self.assertTrue(re.search("123.*-", '123abc-')) @@ -2334,6 +2337,27 @@ def test_possesive_repeat(self): 14. SUCCESS ''') + def test_repeat_index(self): + self.assertEqual(get_debug_out(r'(?:ab)*?(?:cd)*'), '''\ +MIN_REPEAT 0 MAXREPEAT + LITERAL 97 + LITERAL 98 +MAX_REPEAT 0 MAXREPEAT + LITERAL 99 + LITERAL 100 + + 0. INFO 4 0b0 0 MAXREPEAT (to 5) + 5: REPEAT 8 0 MAXREPEAT 0 (to 14) +10. LITERAL 0x61 ('a') +12. LITERAL 0x62 ('b') +14: MIN_UNTIL +15. REPEAT 8 0 MAXREPEAT 1 (to 24) +20. LITERAL 0x63 ('c') +22. LITERAL 0x64 ('d') +24: MAX_UNTIL +25. SUCCESS +''') + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/Misc/NEWS.d/next/Library/2022-04-03-13-19-08.bpo-23689.TFSc3E.rst b/Misc/NEWS.d/next/Library/2022-04-03-13-19-08.bpo-23689.TFSc3E.rst new file mode 100644 index 0000000000000..1032087d9b850 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-03-13-19-08.bpo-23689.TFSc3E.rst @@ -0,0 +1,2 @@ +:mod:`re` module: fix memory leak when a match is terminated by a signal or +memory allocation failure. Patch by Ma Lin. diff --git a/Modules/_sre.c b/Modules/_sre.c index 48193f82475a4..506363d6fbf6d 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -427,6 +427,12 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->lastmark = -1; state->lastindex = -1; + state->repeats_array = PyMem_New(SRE_REPEAT, pattern->repeat_count); + if (!state->repeats_array) { + PyErr_NoMemory(); + goto err; + } + state->buffer.buf = NULL; ptr = getstring(string, &length, &isbytes, &charsize, &state->buffer); if (!ptr) @@ -476,6 +482,9 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, safely casted to `void*`, see bpo-39943 for details. */ PyMem_Free((void*) state->mark); state->mark = NULL; + PyMem_Free(state->repeats_array); + state->repeats_array = NULL; + if (state->buffer.buf) PyBuffer_Release(&state->buffer); return NULL; @@ -491,6 +500,8 @@ state_fini(SRE_STATE* state) /* See above PyMem_Del for why we explicitly cast here. */ PyMem_Free((void*) state->mark); state->mark = NULL; + PyMem_Free(state->repeats_array); + state->repeats_array = NULL; } /* calculate offset from start of string */ @@ -1407,14 +1418,15 @@ _sre.compile groups: Py_ssize_t groupindex: object(subclass_of='&PyDict_Type') indexgroup: object(subclass_of='&PyTuple_Type') + repeat_count: Py_ssize_t [clinic start generated code]*/ static PyObject * _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, PyObject *code, Py_ssize_t groups, PyObject *groupindex, - PyObject *indexgroup) -/*[clinic end generated code: output=ef9c2b3693776404 input=0a68476dbbe5db30]*/ + PyObject *indexgroup, Py_ssize_t repeat_count) +/*[clinic end generated code: output=922af562d51b1657 input=77e39c322501ec2a]*/ { /* "compile" pattern descriptor to pattern object */ @@ -1472,8 +1484,8 @@ _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, self->pattern = pattern; self->flags = flags; - self->groups = groups; + self->repeat_count = repeat_count; if (PyDict_GET_SIZE(groupindex) > 0) { Py_INCREF(groupindex); @@ -1645,7 +1657,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) } static int -_validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +_validate_inner(SRE_CODE *code, SRE_CODE *end, PatternObject *self) { /* Some variables are manipulated by the macros above */ SRE_CODE op; @@ -1666,8 +1678,8 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) sre_match() code is robust even if they don't, and the worst you can get is nonsensical match results. */ GET_ARG; - if (arg > 2 * (size_t)groups + 1) { - VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); + if (arg > 2 * (size_t)self->groups + 1) { + VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)self->groups)); FAIL; } break; @@ -1796,7 +1808,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) if (skip == 0) break; /* Stop 2 before the end; we check the JUMP below */ - if (!_validate_inner(code, code+skip-3, groups)) + if (!_validate_inner(code, code+skip-3, self)) FAIL; code += skip-3; /* Check that it ends with a JUMP, and that each JUMP @@ -1825,7 +1837,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) FAIL; if (max > SRE_MAXREPEAT) FAIL; - if (!_validate_inner(code, code+skip-4, groups)) + if (!_validate_inner(code, code+skip-4, self)) FAIL; code += skip-4; GET_OP; @@ -1837,7 +1849,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) case SRE_OP_REPEAT: case SRE_OP_POSSESSIVE_REPEAT: { - SRE_CODE op1 = op, min, max; + SRE_CODE op1 = op, min, max, repeat_index; GET_SKIP; GET_ARG; min = arg; GET_ARG; max = arg; @@ -1845,9 +1857,17 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) FAIL; if (max > SRE_MAXREPEAT) FAIL; - if (!_validate_inner(code, code+skip-3, groups)) + if (op1 == SRE_OP_REPEAT) { + GET_ARG; repeat_index = arg; + if (repeat_index >= (size_t)self->repeat_count) + FAIL; + skip -= 4; + } else { + skip -= 3; + } + if (!_validate_inner(code, code+skip, self)) FAIL; - code += skip-3; + code += skip; GET_OP; if (op1 == SRE_OP_POSSESSIVE_REPEAT) { if (op != SRE_OP_SUCCESS) @@ -1863,7 +1883,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) case SRE_OP_ATOMIC_GROUP: { GET_SKIP; - if (!_validate_inner(code, code+skip-2, groups)) + if (!_validate_inner(code, code+skip-2, self)) FAIL; code += skip-2; GET_OP; @@ -1877,7 +1897,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) case SRE_OP_GROUPREF_UNI_IGNORE: case SRE_OP_GROUPREF_LOC_IGNORE: GET_ARG; - if (arg >= (size_t)groups) + if (arg >= (size_t)self->groups) FAIL; break; @@ -1886,7 +1906,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) 'group' is either an integer group number or a group name, 'then' and 'else' are sub-regexes, and 'else' is optional. */ GET_ARG; - if (arg >= (size_t)groups) + if (arg >= (size_t)self->groups) FAIL; GET_SKIP_ADJ(1); code--; /* The skip is relative to the first arg! */ @@ -1919,17 +1939,17 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) code[skip-3] == SRE_OP_JUMP) { VTRACE(("both then and else parts present\n")); - if (!_validate_inner(code+1, code+skip-3, groups)) + if (!_validate_inner(code+1, code+skip-3, self)) FAIL; code += skip-2; /* Position after JUMP, at <skipno> */ GET_SKIP; - if (!_validate_inner(code, code+skip-1, groups)) + if (!_validate_inner(code, code+skip-1, self)) FAIL; code += skip-1; } else { VTRACE(("only a then part present\n")); - if (!_validate_inner(code+1, code+skip-1, groups)) + if (!_validate_inner(code+1, code+skip-1, self)) FAIL; code += skip-1; } @@ -1943,7 +1963,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) if (arg & 0x80000000) FAIL; /* Width too large */ /* Stop 1 before the end; we check the SUCCESS below */ - if (!_validate_inner(code+1, code+skip-2, groups)) + if (!_validate_inner(code+1, code+skip-2, self)) FAIL; code += skip-2; GET_OP; @@ -1962,18 +1982,19 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) } static int -_validate_outer(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +_validate_outer(SRE_CODE *code, SRE_CODE *end, PatternObject *self) { - if (groups < 0 || (size_t)groups > SRE_MAXGROUPS || + if (self->groups < 0 || (size_t)self->groups > SRE_MAXGROUPS || + self->repeat_count < 0 || code >= end || end[-1] != SRE_OP_SUCCESS) FAIL; - return _validate_inner(code, end-1, groups); + return _validate_inner(code, end-1, self); } static int _validate(PatternObject *self) { - if (!_validate_outer(self->code, self->code+self->codesize, self->groups)) + if (!_validate_outer(self->code, self->code+self->codesize, self)) { PyErr_SetString(PyExc_RuntimeError, "invalid SRE code"); return 0; diff --git a/Modules/clinic/_sre.c.h b/Modules/clinic/_sre.c.h index 72d772c289ae8..34cbe21f14071 100644 --- a/Modules/clinic/_sre.c.h +++ b/Modules/clinic/_sre.c.h @@ -544,7 +544,7 @@ PyDoc_STRVAR(_sre_SRE_Pattern___deepcopy____doc__, PyDoc_STRVAR(_sre_compile__doc__, "compile($module, /, pattern, flags, code, groups, groupindex,\n" -" indexgroup)\n" +" indexgroup, repeat_count)\n" "--\n" "\n"); @@ -554,23 +554,24 @@ PyDoc_STRVAR(_sre_compile__doc__, static PyObject * _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, PyObject *code, Py_ssize_t groups, PyObject *groupindex, - PyObject *indexgroup); + PyObject *indexgroup, Py_ssize_t repeat_count); static PyObject * _sre_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"pattern", "flags", "code", "groups", "groupindex", "indexgroup", NULL}; + static const char * const _keywords[] = {"pattern", "flags", "code", "groups", "groupindex", "indexgroup", "repeat_count", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "compile", 0}; - PyObject *argsbuf[6]; + PyObject *argsbuf[7]; PyObject *pattern; int flags; PyObject *code; Py_ssize_t groups; PyObject *groupindex; PyObject *indexgroup; + Py_ssize_t repeat_count; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 7, 7, 0, argsbuf); if (!args) { goto exit; } @@ -606,7 +607,19 @@ _sre_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } indexgroup = args[5]; - return_value = _sre_compile_impl(module, pattern, flags, code, groups, groupindex, indexgroup); + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[6]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + repeat_count = ival; + } + return_value = _sre_compile_impl(module, pattern, flags, code, groups, groupindex, indexgroup, repeat_count); exit: return return_value; @@ -910,4 +923,4 @@ _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const exit: return return_value; } -/*[clinic end generated code: output=518f7bb775c1184f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d7510a57a157a38 input=a9049054013a1b77]*/ diff --git a/Modules/sre.h b/Modules/sre.h index 785adbd003e7f..e2c5277aefb5d 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -29,6 +29,8 @@ typedef struct { Py_ssize_t groups; /* must be first! */ PyObject* groupindex; /* dict */ PyObject* indexgroup; /* tuple */ + /* the number of REPEATs */ + Py_ssize_t repeat_count; /* compatibility */ PyObject* pattern; /* pattern source (or None) */ int flags; /* flags used when compiling pattern source */ @@ -83,6 +85,8 @@ typedef struct { size_t data_stack_base; /* current repeat context */ SRE_REPEAT *repeat; + /* repeat contexts array */ + SRE_REPEAT *repeats_array; } SRE_STATE; typedef struct { diff --git a/Modules/sre_constants.h b/Modules/sre_constants.h index 45395dcea807a..8b249493bd5cd 100644 --- a/Modules/sre_constants.h +++ b/Modules/sre_constants.h @@ -11,7 +11,7 @@ * See the _sre.c file for information on usage and redistribution. */ -#define SRE_MAGIC 20220318 +#define SRE_MAGIC 20220402 #define SRE_OP_FAILURE 0 #define SRE_OP_SUCCESS 1 #define SRE_OP_ANY 2 diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 8e4e714eada38..1cc926d956c63 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1032,16 +1032,14 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) case SRE_OP_REPEAT: /* create repeat context. all the hard work is done by the UNTIL operator (MAX_UNTIL, MIN_UNTIL) */ - /* <REPEAT> <skip> <1=min> <2=max> item <UNTIL> tail */ - TRACE(("|%p|%p|REPEAT %d %d\n", ctx->pattern, ctx->ptr, - ctx->pattern[1], ctx->pattern[2])); + /* <REPEAT> <skip> <1=min> <2=max> + <3=repeat_index> item <UNTIL> tail */ + TRACE(("|%p|%p|REPEAT %d %d %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[1], ctx->pattern[2], ctx->pattern[3])); + + /* install repeat context */ + ctx->u.rep = &state->repeats_array[ctx->pattern[3]]; - /* install new repeat context */ - ctx->u.rep = (SRE_REPEAT*) PyObject_Malloc(sizeof(*ctx->u.rep)); - if (!ctx->u.rep) { - PyErr_NoMemory(); - RETURN_FAILURE; - } ctx->u.rep->count = -1; ctx->u.rep->pattern = ctx->pattern; ctx->u.rep->prev = state->repeat; @@ -1051,7 +1049,6 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ctx->ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, ctx->pattern+ctx->pattern[0]); state->repeat = ctx->u.rep->prev; - PyObject_Free(ctx->u.rep); if (ret) { RETURN_ON_ERROR(ret); @@ -1061,7 +1058,8 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) case SRE_OP_MAX_UNTIL: /* maximizing repeat */ - /* <REPEAT> <skip> <1=min> <2=max> item <MAX_UNTIL> tail */ + /* <REPEAT> <skip> <1=min> <2=max> + <3=repeat_index> item <MAX_UNTIL> tail */ /* FIXME: we probably need to deal with zero-width matches in here... */ @@ -1081,7 +1079,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, - ctx->u.rep->pattern+3); + ctx->u.rep->pattern+4); if (ret) { RETURN_ON_ERROR(ret); RETURN_SUCCESS; @@ -1103,7 +1101,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DATA_PUSH(&ctx->u.rep->last_ptr); ctx->u.rep->last_ptr = state->ptr; DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2, - ctx->u.rep->pattern+3); + ctx->u.rep->pattern+4); DATA_POP(&ctx->u.rep->last_ptr); if (ret) { MARK_POP_DISCARD(ctx->lastmark); @@ -1128,7 +1126,8 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) case SRE_OP_MIN_UNTIL: /* minimizing repeat */ - /* <REPEAT> <skip> <1=min> <2=max> item <MIN_UNTIL> tail */ + /* <REPEAT> <skip> <1=min> <2=max> + <3=repeat_index> item <MIN_UNTIL> tail */ ctx->u.rep = state->repeat; if (!ctx->u.rep) @@ -1145,7 +1144,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, - ctx->u.rep->pattern+3); + ctx->u.rep->pattern+4); if (ret) { RETURN_ON_ERROR(ret); RETURN_SUCCESS; @@ -1188,7 +1187,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DATA_PUSH(&ctx->u.rep->last_ptr); ctx->u.rep->last_ptr = state->ptr; DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3, - ctx->u.rep->pattern+3); + ctx->u.rep->pattern+4); DATA_POP(&ctx->u.rep->last_ptr); if (ret) { RETURN_ON_ERROR(ret);

1 0

bpo-47205: Skip error check of sched_get/setaffinity on FreeBSD (GH-32285)
by miss-islington April 3, 2022

April 3, 2022

https://github.com/python/cpython/commit/490ccbd6e0e5aad07a40c79a6b0c45ffca… commit: 490ccbd6e0e5aad07a40c79a6b0c45ffca91724b branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-03T08:30:39-07:00 summary: bpo-47205: Skip error check of sched_get/setaffinity on FreeBSD (GH-32285) (cherry picked from commit b82cdd1dac9a9be52051abd90a1ce69236ac41f4) Co-authored-by: Christian Heimes <christian(a)python.org> files: A Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst M Lib/test/test_posix.py diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index a8db30679af8d..6ba14547bdd6c 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1153,7 +1153,9 @@ def test_sched_getaffinity(self): mask = posix.sched_getaffinity(0) self.assertIsInstance(mask, set) self.assertGreaterEqual(len(mask), 1) - self.assertRaises(OSError, posix.sched_getaffinity, -1) + if not sys.platform.startswith("freebsd"): + # bpo-47205: does not raise OSError on FreeBSD + self.assertRaises(OSError, posix.sched_getaffinity, -1) for cpu in mask: self.assertIsInstance(cpu, int) self.assertGreaterEqual(cpu, 0) @@ -1171,7 +1173,9 @@ def test_sched_setaffinity(self): self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) - self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) + if not sys.platform.startswith("freebsd"): + # bpo-47205: does not raise OSError on FreeBSD + self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) def test_rtld_constants(self): # check presence of major RTLD_* constants diff --git a/Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst b/Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst new file mode 100644 index 0000000000000..35fd94421326e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-04-03-14-38-21.bpo-47205.hbbTnh.rst @@ -0,0 +1,2 @@ +Skip test for :func:`~os.sched_getaffinity` and +:func:`~os.sched_setaffinity` error case on FreeBSD.

1 0