Mailman 3 April 2022 - Python-checkins

bpo-4833: Add ZipFile.mkdir (GH-32160)
by ethanfurman April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/050a8f94c678a05d506fe192c863c4a572… commit: 050a8f94c678a05d506fe192c863c4a572178c42 branch: main author: Sam Ezeh <sam.z.ezeh(a)gmail.com> committer: ethanfurman <ethan(a)stoneleaf.us> date: 2022-04-05T11:41:38-07:00 summary: bpo-4833: Add ZipFile.mkdir (GH-32160) files: A Misc/NEWS.d/next/Library/2022-03-28-20-16-37.bpo-4833.2vSUE5.rst M Doc/library/zipfile.rst M Lib/test/test_zipfile.py M Lib/zipfile.py diff --git a/Doc/library/zipfile.… [View More]rst b/Doc/library/zipfile.rst index bfcc883de6927..d6a1fce49c545 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -478,6 +478,17 @@ ZipFile Objects a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. +.. method:: ZipFile.mkdir(zinfo_or_directory, mode=511) + + Create a directory inside the archive. If *zinfo_or_directory* is a string, + a directory is created inside the archive with the mode that is specified in + the *mode* argument. If, however, *zinfo_or_directory* is + a :class:`ZipInfo` instance then the *mode* argument is ignored. + + The archive must be opened with mode ``'w'``, ``'x'`` or ``'a'``. + + .. versionadded:: 3.11 + The following data attributes are also available: diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 26c40457e62a0..17111b3a40fef 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2637,6 +2637,59 @@ def test_writestr_dir(self): self.assertTrue(os.path.isdir(os.path.join(target, "x"))) self.assertEqual(os.listdir(target), ["x"]) + def test_mkdir(self): + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.mkdir("directory") + zinfo = zf.filelist[0] + self.assertEqual(zinfo.filename, "directory/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + zf.mkdir("directory2/") + zinfo = zf.filelist[1] + self.assertEqual(zinfo.filename, "directory2/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + zf.mkdir("directory3", mode=0o777) + zinfo = zf.filelist[2] + self.assertEqual(zinfo.filename, "directory3/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + old_zinfo = zipfile.ZipInfo("directory4/") + old_zinfo.external_attr = (0o40777 << 16) | 0x10 + old_zinfo.CRC = 0 + old_zinfo.file_size = 0 + old_zinfo.compress_size = 0 + zf.mkdir(old_zinfo) + new_zinfo = zf.filelist[3] + self.assertEqual(old_zinfo.filename, "directory4/") + self.assertEqual(old_zinfo.external_attr, new_zinfo.external_attr) + + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zf.extractall(target) + self.assertEqual(set(os.listdir(target)), {"directory", "directory2", "directory3", "directory4"}) + + def test_create_directory_with_write(self): + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.writestr(zipfile.ZipInfo('directory/'), '') + + zinfo = zf.filelist[0] + self.assertEqual(zinfo.filename, "directory/") + + directory = os.path.join(TESTFN2, "directory2") + os.mkdir(directory) + mode = os.stat(directory).st_mode + zf.write(directory, arcname="directory2/") + zinfo = zf.filelist[1] + self.assertEqual(zinfo.filename, "directory2/") + self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) + + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zf.extractall(target) + + self.assertEqual(set(os.listdir(target)), {"directory", "directory2"}) + def tearDown(self): rmtree(TESTFN2) if os.path.exists(TESTFN): diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 721834aff13a7..dc02011084329 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1772,6 +1772,7 @@ def write(self, filename, arcname=None, if zinfo.is_dir(): zinfo.compress_size = 0 zinfo.CRC = 0 + self.mkdir(zinfo) else: if compress_type is not None: zinfo.compress_type = compress_type @@ -1783,23 +1784,6 @@ def write(self, filename, arcname=None, else: zinfo._compresslevel = self.compresslevel - if zinfo.is_dir(): - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header bytes - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 - - self._writecheck(zinfo) - self._didModify = True - - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - self.fp.write(zinfo.FileHeader(False)) - self.start_dir = self.fp.tell() - else: with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: shutil.copyfileobj(src, dest, 1024*8) @@ -1844,6 +1828,41 @@ def writestr(self, zinfo_or_arcname, data, with self.open(zinfo, mode='w') as dest: dest.write(data) + def mkdir(self, zinfo_or_directory_name, mode=511): + """Creates a directory inside the zip archive.""" + if isinstance(zinfo_or_directory_name, ZipInfo): + zinfo = zinfo_or_directory_name + if not zinfo.is_dir(): + raise ValueError("The given ZipInfo does not describe a directory") + elif isinstance(zinfo_or_directory_name, str): + directory_name = zinfo_or_directory_name + if not directory_name.endswith("/"): + directory_name += "/" + zinfo = ZipInfo(directory_name) + zinfo.compress_size = 0 + zinfo.CRC = 0 + zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16 + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 + else: + raise TypeError("Expected type str or ZipInfo") + + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() # Start of header bytes + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 + + self._writecheck(zinfo) + self._didModify = True + + self.filelist.append(zinfo) + self.NameToInfo[zinfo.filename] = zinfo + self.fp.write(zinfo.FileHeader(False)) + self.start_dir = self.fp.tell() + def __del__(self): """Call the "close()" method in case the user forgot.""" self.close() diff --git a/Misc/NEWS.d/next/Library/2022-03-28-20-16-37.bpo-4833.2vSUE5.rst b/Misc/NEWS.d/next/Library/2022-03-28-20-16-37.bpo-4833.2vSUE5.rst new file mode 100644 index 0000000000000..7696091221cb5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-28-20-16-37.bpo-4833.2vSUE5.rst @@ -0,0 +1 @@ +Add :meth:`ZipFile.mkdir` [View Less]

1 0

[3.10] Backport bpo-47212 (GH-32302) to Python 3.10 (GH-32334)
by miss-islington April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/94609e3192f15636c760a48c4c1c2c236f… commit: 94609e3192f15636c760a48c4c1c2c236fac3217 branch: 3.10 author: Matthieu Dartiailh <marul(a)laposte.net> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-05T09:21:49-07:00 summary: [3.10] Backport bpo-47212 (GH-32302) to Python 3.10 (GH-32334) (cherry picked from commit aa0f056a00c4bcaef83d729e042359ddae903382) # Conflicts: # Grammar/python.gram # … [View More]Parser/action_helpers.c Automerge-Triggered-By: GH:pablogsal files: A Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst M Grammar/python.gram M Lib/test/test_exceptions.py M Lib/test/test_syntax.py M Parser/parser.c M Parser/pegen.c M Parser/pegen.h diff --git a/Grammar/python.gram b/Grammar/python.gram index 99e01355a582c..84b9c9b82720b 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -830,12 +830,12 @@ t_lookahead: '(' | '[' | '.' invalid_arguments: | a=args ',' '*' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable argument unpacking follows keyword argument unpacking") } | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, comprehension_ty)->target, "Generator expression must be parenthesized") } + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")} | a=args b=for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a, b) } | args ',' a=expression b=for_if_clauses { - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") } + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } invalid_kwarg: | a[Token*]=('True'|'False'|'None') b='=' { @@ -975,7 +975,7 @@ invalid_finally_stmt: invalid_except_stmt_indent: | a='except' expression ['as' NAME ] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) } - | a='except' ':' NEWLINE !INDENT { RAISE_SYNTAX_ERROR("expected an indented block after except statement on line %d", a->lineno) } + | a='except' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) } invalid_match_stmt: | "match" subject_expr !':' { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) } | a="match" subject=subject_expr ':' NEWLINE !INDENT { diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 2bdd7214b0505..d9ea8a4872d71 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -198,12 +198,17 @@ def ckmsg(src, msg, exception=SyntaxError): s = '''if True:\n print()\n\texec "mixed tabs and spaces"''' ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError) - def check(self, src, lineno, offset, encoding='utf-8'): + def check(self, src, lineno, offset, end_lineno=None, end_offset=None, encoding='utf-8'): with self.subTest(source=src, lineno=lineno, offset=offset): with self.assertRaises(SyntaxError) as cm: compile(src, '<fragment>', 'exec') self.assertEqual(cm.exception.lineno, lineno) self.assertEqual(cm.exception.offset, offset) + if end_lineno is not None: + self.assertEqual(cm.exception.end_lineno, end_lineno) + if end_offset is not None: + self.assertEqual(cm.exception.end_offset, end_offset) + if cm.exception.text is not None: if not isinstance(src, str): src = src.decode(encoding, 'replace') @@ -235,6 +240,10 @@ def testSyntaxErrorOffset(self): check('match ...:\n case {**rest, "key": value}:\n ...', 2, 19) check("[a b c d e f]", 1, 2) check("for x yfff:", 1, 7) + check("f(a for a in b, c)", 1, 3, 1, 15) + check("f(a for a in b if a, c)", 1, 3, 1, 20) + check("f(a, b for b in c)", 1, 6, 1, 18) + check("f(a, b for b in c, d)", 1, 6, 1, 18) # Errors thrown by compile.c check('class foo:return 1', 1, 11) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 09c0d5682a7e4..006374e7cd912 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1012,6 +1012,13 @@ Traceback (most recent call last): IndentationError: expected an indented block after 'try' statement on line 1 + >>> try: + ... something() + ... except: + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after 'except' statement on line 3 + >>> try: ... something() ... except A: diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst b/Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst new file mode 100644 index 0000000000000..8f1f6b6cfbbb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst @@ -0,0 +1,3 @@ +Raise :exc:`IndentationError` instead of :exc:`SyntaxError` for a bare +``except`` with no following indent. Improve :exc:`SyntaxError` locations for +an un-parenthesized generator used as arguments. Patch by Matthieu Dartiailh. diff --git a/Parser/parser.c b/Parser/parser.c index 3f73003b78973..862950018a312 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -18419,7 +18419,7 @@ invalid_arguments_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); - _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , PyPegen_last_item ( b , comprehension_ty ) -> target , "Generator expression must be parenthesized" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , _PyPegen_get_last_comprehension_item ( PyPegen_last_item ( b , comprehension_ty ) ) , "Generator expression must be parenthesized" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -18512,7 +18512,7 @@ invalid_arguments_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ',' expression for_if_clauses")); - _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , asdl_seq_GET ( b , b -> size - 1 ) -> target , "Generator expression must be parenthesized" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , _PyPegen_get_last_comprehension_item ( PyPegen_last_item ( b , comprehension_ty ) ) , "Generator expression must be parenthesized" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -20804,7 +20804,7 @@ invalid_except_stmt_indent_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_except_stmt_indent[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' ':' NEWLINE !INDENT")); - _res = RAISE_SYNTAX_ERROR ( "expected an indented block after except statement on line %d" , a -> lineno ); + _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'except' statement on line %d" , a -> lineno ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen.c b/Parser/pegen.c index 90a6ab98cf42e..c04824315c7a3 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -2568,7 +2568,7 @@ void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) { } -static inline expr_ty +expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension) { if (comprehension->ifs == NULL || asdl_seq_LEN(comprehension->ifs) == 0) { return comprehension->iter; diff --git a/Parser/pegen.h b/Parser/pegen.h index 0e5a057c1f196..118fbc7b3b78a 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -327,6 +327,7 @@ _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e) } void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); +expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension); void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions); [View Less]

1 0

bpo-47009: Fix assert on big endian (GH-32332)
by tiran April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/9e88b572fb904b172f9e344069fb7118f1… commit: 9e88b572fb904b172f9e344069fb7118f1cee517 branch: main author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-04-05T18:18:02+02:00 summary: bpo-47009: Fix assert on big endian (GH-32332) files: M Python/ceval.c diff --git a/Python/ceval.c b/Python/ceval.c index 9b7c42cbe4b11..487e09bc66417 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5032,7 +… [View More]

1 0

bpo-46607: Add DeprecationWarning for LegacyInterpolation, deprecated in docs since 3.2 (GH-30927)
by gpshead April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/75280944e5ca957eec7f814b9d0608fc84… commit: 75280944e5ca957eec7f814b9d0608fc84fc5811 branch: main author: Hugo van Kemenade <hugovk(a)users.noreply.github.com> committer: gpshead <greg(a)krypto.org> date: 2022-04-05T08:15:11-07:00 summary: bpo-46607: Add DeprecationWarning for LegacyInterpolation, deprecated in docs since 3.2 (GH-30927) files: A Misc/NEWS.d/next/Library/2022-01-26-18-30-34.bpo-46607.xnhT4a.rst M Doc/whatsnew/3.11.rst M Lib/… [View More]configparser.py M Lib/test/test_configparser.py diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index c312645c31cd7..537fa49fd2f28 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -590,6 +590,12 @@ Deprecated (Contributed by Hugo van Kemenade in :issue:`45173`.) +* :class:`configparser.LegacyInterpolation` has been deprecated in the docstring + since Python 3.2. It now emits a :exc:`DeprecationWarning` and will be removed + in Python 3.13. Use :class:`configparser.BasicInterpolation` or + :class:`configparser.ExtendedInterpolation instead. + (Contributed by Hugo van Kemenade in :issue:`46607`.) + * The :func:`locale.getdefaultlocale` function is deprecated and will be removed in Python 3.13. Use :func:`locale.setlocale`, :func:`locale.getpreferredencoding(False) <locale.getpreferredencoding>` and diff --git a/Lib/configparser.py b/Lib/configparser.py index f5666f518f121..de9ee537ac188 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -525,6 +525,15 @@ class LegacyInterpolation(Interpolation): _KEYCRE = re.compile(r"%$([^)]*)$s|.") + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + "LegacyInterpolation has been deprecated since Python 3.2 " + "and will be removed from the configparser module in Python 3.13. " + "Use BasicInterpolation or ExtendedInterpolation instead.", + DeprecationWarning, stacklevel=2 + ) + def before_get(self, parser, section, option, value, vars): rawval = value depth = MAX_INTERPOLATION_DEPTH diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 569959c3ed919..efd98ffb67a54 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -1666,6 +1666,14 @@ def test_safeconfigparser_deprecation(self): for warning in w: self.assertTrue(warning.category is DeprecationWarning) + def test_legacyinterpolation_deprecation(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always", DeprecationWarning) + configparser.LegacyInterpolation() + self.assertGreaterEqual(len(w), 1) + for warning in w: + self.assertIs(warning.category, DeprecationWarning) + def test_sectionproxy_repr(self): parser = configparser.ConfigParser() parser.read_string(""" diff --git a/Misc/NEWS.d/next/Library/2022-01-26-18-30-34.bpo-46607.xnhT4a.rst b/Misc/NEWS.d/next/Library/2022-01-26-18-30-34.bpo-46607.xnhT4a.rst new file mode 100644 index 0000000000000..e0c7ed0531b57 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-26-18-30-34.bpo-46607.xnhT4a.rst @@ -0,0 +1,3 @@ +Add :exc:`DeprecationWarning` to :class:`LegacyInterpolation`, deprecated in +the docstring since Python 3.2. Will be removed in Python 3.13. Use +:class:`BasicInterpolation` or :class:`ExtendedInterpolation` instead. [View Less]

1 0

bpo-47088: Add typing.LiteralString (PEP 675) (GH-32064)
by JelleZijlstra April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/cfb849a326e52a4edc577112ebf60e1d9d… commit: cfb849a326e52a4edc577112ebf60e1d9d0d7fdb branch: main author: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2022-04-05T07:21:03-07:00 summary: bpo-47088: Add typing.LiteralString (PEP 675) (GH-32064) Co-authored-by: Nick Pope <nick(a)nickpope.me.uk> files: A Misc/NEWS.d/next/Library/2022-03-22-19-18-31.bpo-47088.JM1kNI.rst M Doc/… [View More]library/typing.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 0a4e848c67736..fdd00a277b507 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -76,6 +76,8 @@ annotations. These include: *Introducing* :data:`TypeGuard` * :pep:`673`: Self type *Introducing* :data:`Self` +* :pep:`675`: Arbitrary Literal String Type + *Introducing* :data:`LiteralString` .. _type-aliases: @@ -585,6 +587,33 @@ These can be used as types in annotations and do not support ``[]``. avoiding type checker errors with classes that can duck type anywhere or are highly dynamic. +.. data:: LiteralString + + Special type that includes only literal strings. A string + literal is compatible with ``LiteralString``, as is another + ``LiteralString``, but an object typed as just ``str`` is not. + A string created by composing ``LiteralString``-typed objects + is also acceptable as a ``LiteralString``. + + Example:: + + def run_query(sql: LiteralString) -> ... + ... + + def caller(arbitrary_string: str, literal_string: LiteralString) -> None: + run_query("SELECT * FROM students") # ok + run_query(literal_string) # ok + run_query("SELECT * FROM " + literal_string) # ok + run_query(arbitrary_string) # type checker error + run_query( # type checker error + f"SELECT * FROM students WHERE name = {arbitrary_string}" + ) + + This is useful for sensitive APIs where arbitrary user-generated + strings could generate problems. For example, the two cases above + that generate type checker errors could be vulnerable to an SQL + injection attack. + .. data:: Never The `bottom type <https://en.wikipedia.org/wiki/Bottom_type>`_, diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 041b6ad9ed6dd..e09f8aa3fb849 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -27,7 +27,7 @@ from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef -from typing import Self +from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs from typing import TypeGuard @@ -265,6 +265,60 @@ def test_alias(self): self.assertEqual(get_args(alias_3), (Self,)) +class LiteralStringTests(BaseTestCase): + def test_equality(self): + self.assertEqual(LiteralString, LiteralString) + self.assertIs(LiteralString, LiteralString) + self.assertNotEqual(LiteralString, None) + + def test_basics(self): + class Foo: + def bar(self) -> LiteralString: ... + class FooStr: + def bar(self) -> 'LiteralString': ... + class FooStrTyping: + def bar(self) -> 'typing.LiteralString': ... + + for target in [Foo, FooStr, FooStrTyping]: + with self.subTest(target=target): + self.assertEqual(gth(target.bar), {'return': LiteralString}) + self.assertIs(get_origin(LiteralString), None) + + def test_repr(self): + self.assertEqual(repr(LiteralString), 'typing.LiteralString') + + def test_cannot_subscript(self): + with self.assertRaises(TypeError): + LiteralString[int] + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class C(type(LiteralString)): + pass + with self.assertRaises(TypeError): + class C(LiteralString): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + LiteralString() + with self.assertRaises(TypeError): + type(LiteralString)() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, LiteralString) + with self.assertRaises(TypeError): + issubclass(int, LiteralString) + + def test_alias(self): + alias_1 = Tuple[LiteralString, LiteralString] + alias_2 = List[LiteralString] + alias_3 = ClassVar[LiteralString] + self.assertEqual(get_args(alias_1), (LiteralString, LiteralString)) + self.assertEqual(get_args(alias_2), (LiteralString,)) + self.assertEqual(get_args(alias_3), (LiteralString,)) + class TypeVarTests(BaseTestCase): def test_basic_plain(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index 4636798bd6956..26c6b8c278b73 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -126,6 +126,7 @@ def _idfunc(_, x): 'get_origin', 'get_type_hints', 'is_typeddict', + 'LiteralString', 'Never', 'NewType', 'no_type_check', @@ -180,7 +181,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") - if arg in (Any, NoReturn, Never, Self, TypeAlias): + if arg in (Any, LiteralString, NoReturn, Never, Self, TypeAlias): return arg if allow_special_forms and arg in (ClassVar, Final): return arg @@ -523,6 +524,34 @@ def returns_self(self) -> Self: raise TypeError(f"{self} is not subscriptable") +@_SpecialForm +def LiteralString(self, parameters): + """Represents an arbitrary literal string. + + Example:: + + from typing import LiteralString + + def run_query(sql: LiteralString) -> ... + ... + + def caller(arbitrary_string: str, literal_string: LiteralString) -> None: + run_query("SELECT * FROM students") # ok + run_query(literal_string) # ok + run_query("SELECT * FROM " + literal_string) # ok + run_query(arbitrary_string) # type checker error + run_query( # type checker error + f"SELECT * FROM students WHERE name = {arbitrary_string}" + ) + + Only string literals and other LiteralStrings are compatible + with LiteralString. This provides a tool to help prevent + security issues such as SQL injection. + + """ + raise TypeError(f"{self} is not subscriptable") + + @_SpecialForm def ClassVar(self, parameters): """Special type construct to mark class variables. diff --git a/Misc/NEWS.d/next/Library/2022-03-22-19-18-31.bpo-47088.JM1kNI.rst b/Misc/NEWS.d/next/Library/2022-03-22-19-18-31.bpo-47088.JM1kNI.rst new file mode 100644 index 0000000000000..10a814e018245 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-22-19-18-31.bpo-47088.JM1kNI.rst @@ -0,0 +1,2 @@ +Implement :data:`typing.LiteralString`, part of :pep:`675`. Patch by Jelle +Zijlstra. [View Less]

1 0

bpo-41930: Add support for SQLite serialise/deserialise API (GH-26728)
by JelleZijlstra April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/a7551247e7cb7010fb4735281f1afa4abe… commit: a7551247e7cb7010fb4735281f1afa4abeb8a9cc branch: main author: Erlend Egeberg Aasland <erlend.aasland(a)innova.no> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2022-04-05T07:15:25-07:00 summary: bpo-41930: Add support for SQLite serialise/deserialise API (GH-26728) Co-authored-by: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> Co-authored-by: Kumar Aditya <59607654+… [View More]kumaraditya303(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst M Doc/library/sqlite3.rst M Doc/whatsnew/3.11.rst M Lib/test/test_sqlite3/test_dbapi.py M Modules/_sqlite/clinic/connection.c.h M Modules/_sqlite/connection.c M PCbuild/_sqlite3.vcxproj M configure M configure.ac M pyconfig.h.in diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e70d038e61d82..852b68437a265 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -748,6 +748,44 @@ Connection Objects .. versionadded:: 3.11 + .. method:: serialize(*, name="main") + + This method serializes a database into a :class:`bytes` object. For an + ordinary on-disk database file, the serialization is just a copy of the + disk file. For an in-memory database or a "temp" database, the + serialization is the same sequence of bytes which would be written to + disk if that database were backed up to disk. + + *name* is the database to be serialized, and defaults to the main + database. + + .. note:: + + This method is only available if the underlying SQLite library has the + serialize API. + + .. versionadded:: 3.11 + + + .. method:: deserialize(data, /, *, name="main") + + This method causes the database connection to disconnect from database + *name*, and reopen *name* as an in-memory database based on the + serialization contained in *data*. Deserialization will raise + :exc:`OperationalError` if the database connection is currently involved + in a read transaction or a backup operation. :exc:`DataError` will be + raised if ``len(data)`` is larger than ``2**63 - 1``, and + :exc:`DatabaseError` will be raised if *data* does not contain a valid + SQLite database. + + .. note:: + + This method is only available if the underlying SQLite library has the + deserialize API. + + .. versionadded:: 3.11 + + .. _sqlite3-cursor-objects: Cursor Objects diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index d0c10a9100997..c312645c31cd7 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -366,6 +366,11 @@ sqlite3 Instead we leave it to the SQLite library to handle these cases. (Contributed by Erlend E. Aasland in :issue:`44092`.) +* Add :meth:`~sqlite3.Connection.serialize` and + :meth:`~sqlite3.Connection.deserialize` to :class:`sqlite3.Connection` for + serializing and deserializing databases. + (Contributed by Erlend E. Aasland in :issue:`41930`.) + sys --- diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 177c2cd327ff3..02482816cb933 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -29,6 +29,7 @@ from test.support import ( SHORT_TIMEOUT, + bigmemtest, check_disallow_instantiation, threading_helper, ) @@ -603,6 +604,56 @@ def test_uninit_operations(self): func) +(a)unittest.skipUnless(hasattr(sqlite.Connection, "serialize"), + "Needs SQLite serialize API") +class SerializeTests(unittest.TestCase): + def test_serialize_deserialize(self): + with memory_database() as cx: + with cx: + cx.execute("create table t(t)") + data = cx.serialize() + self.assertEqual(len(data), 8192) + + # Remove test table, verify that it was removed. + with cx: + cx.execute("drop table t") + regex = "no such table" + with self.assertRaisesRegex(sqlite.OperationalError, regex): + cx.execute("select t from t") + + # Deserialize and verify that test table is restored. + cx.deserialize(data) + cx.execute("select t from t") + + def test_deserialize_wrong_args(self): + dataset = ( + (BufferError, memoryview(b"blob")[::2]), + (TypeError, []), + (TypeError, 1), + (TypeError, None), + ) + for exc, arg in dataset: + with self.subTest(exc=exc, arg=arg): + with memory_database() as cx: + self.assertRaises(exc, cx.deserialize, arg) + + def test_deserialize_corrupt_database(self): + with memory_database() as cx: + regex = "file is not a database" + with self.assertRaisesRegex(sqlite.DatabaseError, regex): + cx.deserialize(b"\0\1\3") + # SQLite does not generate an error until you try to query the + # deserialized database. + cx.execute("create table fail(f)") + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @bigmemtest(size=2**63, memuse=3, dry_run=False) + def test_deserialize_too_much_data_64bit(self): + with memory_database() as cx: + with self.assertRaisesRegex(OverflowError, "'data' is too large"): + cx.deserialize(b"b" * size) + + class OpenTests(unittest.TestCase): _sql = "create table test(id integer)" @@ -1030,6 +1081,10 @@ def test_check_connection_thread(self): lambda: self.con.setlimit(sqlite.SQLITE_LIMIT_LENGTH, -1), lambda: self.con.getlimit(sqlite.SQLITE_LIMIT_LENGTH), ] + if hasattr(sqlite.Connection, "serialize"): + fns.append(lambda: self.con.serialize()) + fns.append(lambda: self.con.deserialize(b"")) + for fn in fns: with self.subTest(fn=fn): self._run_test(fn) diff --git a/Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst b/Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst new file mode 100644 index 0000000000000..ce494e7225e22 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst @@ -0,0 +1,3 @@ +Add :meth:`~sqlite3.Connection.serialize` and +:meth:`~sqlite3.Connection.deserialize` support to :mod:`sqlite3`. Patch by +Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 111e344fd2ae1..99ef94ecd71ec 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -693,6 +693,156 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyTypeObject *cl return return_value; } +#if defined(PY_SQLITE_HAVE_SERIALIZE) + +PyDoc_STRVAR(serialize__doc__, +"serialize($self, /, *, name=\'main\')\n" +"--\n" +"\n" +"Serialize a database into a byte string.\n" +"\n" +" name\n" +" Which database to serialize.\n" +"\n" +"For an ordinary on-disk database file, the serialization is just a copy of the\n" +"disk file. For an in-memory database or a \"temp\" database, the serialization is\n" +"the same sequence of bytes which would be written to disk if that database\n" +"were backed up to disk."); + +#define SERIALIZE_METHODDEF \ + {"serialize", (PyCFunction)(void(*)(void))serialize, METH_FASTCALL|METH_KEYWORDS, serialize__doc__}, + +static PyObject * +serialize_impl(pysqlite_Connection *self, const char *name); + +static PyObject * +serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"name", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "serialize", 0}; + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + const char *name = "main"; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("serialize", "argument 'name'", "str", args[0]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_kwonly: + return_value = serialize_impl(self, name); + +exit: + return return_value; +} + +#endif /* defined(PY_SQLITE_HAVE_SERIALIZE) */ + +#if defined(PY_SQLITE_HAVE_SERIALIZE) + +PyDoc_STRVAR(deserialize__doc__, +"deserialize($self, data, /, *, name=\'main\')\n" +"--\n" +"\n" +"Load a serialized database.\n" +"\n" +" data\n" +" The serialized database content.\n" +" name\n" +" Which database to reopen with the deserialization.\n" +"\n" +"The deserialize interface causes the database connection to disconnect from the\n" +"target database, and then reopen it as an in-memory database based on the given\n" +"serialized data.\n" +"\n" +"The deserialize interface will fail with SQLITE_BUSY if the database is\n" +"currently in a read transaction or is involved in a backup operation."); + +#define DESERIALIZE_METHODDEF \ + {"deserialize", (PyCFunction)(void(*)(void))deserialize, METH_FASTCALL|METH_KEYWORDS, deserialize__doc__}, + +static PyObject * +deserialize_impl(pysqlite_Connection *self, Py_buffer *data, + const char *name); + +static PyObject * +deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "name", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "deserialize", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_buffer data = {NULL, NULL}; + const char *name = "main"; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyUnicode_Check(args[0])) { + Py_ssize_t len; + const char *ptr = PyUnicode_AsUTF8AndSize(args[0], &len); + if (ptr == NULL) { + goto exit; + } + PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + } + else { /* any bytes-like object */ + if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!PyBuffer_IsContiguous(&data, 'C')) { + _PyArg_BadArgument("deserialize", "argument 1", "contiguous buffer", args[0]); + goto exit; + } + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("deserialize", "argument 'name'", "str", args[1]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_kwonly: + return_value = deserialize_impl(self, &data, name); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + + return return_value; +} + +#endif /* defined(PY_SQLITE_HAVE_SERIALIZE) */ + PyDoc_STRVAR(pysqlite_connection_enter__doc__, "__enter__($self, /)\n" "--\n" @@ -832,4 +982,12 @@ getlimit(pysqlite_Connection *self, PyObject *arg) #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=176c9095219b17c4 input=a9049054013a1b77]*/ + +#ifndef SERIALIZE_METHODDEF + #define SERIALIZE_METHODDEF +#endif /* !defined(SERIALIZE_METHODDEF) */ + +#ifndef DESERIALIZE_METHODDEF + #define DESERIALIZE_METHODDEF +#endif /* !defined(DESERIALIZE_METHODDEF) */ +/*[clinic end generated code: output=d965a68f9229a56c input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 37f6d0fa5a502..9d187cfa99d23 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1818,6 +1818,125 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, Py_RETURN_NONE; } +#ifdef PY_SQLITE_HAVE_SERIALIZE +/*[clinic input] +_sqlite3.Connection.serialize as serialize + + * + name: str = "main" + Which database to serialize. + +Serialize a database into a byte string. + +For an ordinary on-disk database file, the serialization is just a copy of the +disk file. For an in-memory database or a "temp" database, the serialization is +the same sequence of bytes which would be written to disk if that database +were backed up to disk. +[clinic start generated code]*/ + +static PyObject * +serialize_impl(pysqlite_Connection *self, const char *name) +/*[clinic end generated code: output=97342b0e55239dd3 input=d2eb5194a65abe2b]*/ +{ + if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { + return NULL; + } + + /* If SQLite has a contiguous memory representation of the database, we can + * avoid memory allocations, so we try with the no-copy flag first. + */ + sqlite3_int64 size; + unsigned int flags = SQLITE_SERIALIZE_NOCOPY; + const char *data; + + Py_BEGIN_ALLOW_THREADS + data = (const char *)sqlite3_serialize(self->db, name, &size, flags); + if (data == NULL) { + flags &= ~SQLITE_SERIALIZE_NOCOPY; + data = (const char *)sqlite3_serialize(self->db, name, &size, flags); + } + Py_END_ALLOW_THREADS + + if (data == NULL) { + PyErr_Format(self->OperationalError, "unable to serialize '%s'", + name); + return NULL; + } + PyObject *res = PyBytes_FromStringAndSize(data, size); + if (!(flags & SQLITE_SERIALIZE_NOCOPY)) { + sqlite3_free((void *)data); + } + return res; +} + +/*[clinic input] +_sqlite3.Connection.deserialize as deserialize + + data: Py_buffer(accept={buffer, str}) + The serialized database content. + / + * + name: str = "main" + Which database to reopen with the deserialization. + +Load a serialized database. + +The deserialize interface causes the database connection to disconnect from the +target database, and then reopen it as an in-memory database based on the given +serialized data. + +The deserialize interface will fail with SQLITE_BUSY if the database is +currently in a read transaction or is involved in a backup operation. +[clinic start generated code]*/ + +static PyObject * +deserialize_impl(pysqlite_Connection *self, Py_buffer *data, + const char *name) +/*[clinic end generated code: output=e394c798b98bad89 input=1be4ca1faacf28f2]*/ +{ + if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { + return NULL; + } + + /* Transfer ownership of the buffer to SQLite: + * - Move buffer from Py to SQLite + * - Tell SQLite to free buffer memory + * - Tell SQLite that it is permitted to grow the resulting database + * + * Make sure we don't overflow sqlite3_deserialize(); it accepts a signed + * 64-bit int as its data size argument. + * + * We can safely use sqlite3_malloc64 here, since it was introduced before + * the serialize APIs. + */ + if (data->len > 9223372036854775807) { // (1 << 63) - 1 + PyErr_SetString(PyExc_OverflowError, "'data' is too large"); + return NULL; + } + + sqlite3_int64 size = (sqlite3_int64)data->len; + unsigned char *buf = sqlite3_malloc64(size); + if (buf == NULL) { + return PyErr_NoMemory(); + } + + const unsigned int flags = SQLITE_DESERIALIZE_FREEONCLOSE | + SQLITE_DESERIALIZE_RESIZEABLE; + int rc; + Py_BEGIN_ALLOW_THREADS + (void)memcpy(buf, data->buf, data->len); + rc = sqlite3_deserialize(self->db, name, buf, size, size, flags); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + (void)_pysqlite_seterror(self->state, self->db); + return NULL; + } + Py_RETURN_NONE; +} +#endif // PY_SQLITE_HAVE_SERIALIZE + + /*[clinic input] _sqlite3.Connection.__enter__ as pysqlite_connection_enter @@ -1971,6 +2090,8 @@ static PyMethodDef connection_methods[] = { PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF SETLIMIT_METHODDEF GETLIMIT_METHODDEF + SERIALIZE_METHODDEF + DESERIALIZE_METHODDEF {NULL, NULL} }; diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index e268c473f4c98..9cff43f73e5be 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -94,6 +94,7 @@ <ItemDefinitionGroup> <ClCompile> <AdditionalIncludeDirectories>$(sqlite3Dir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>PY_SQLITE_HAVE_SERIALIZE;%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> </ItemDefinitionGroup> <ItemGroup> diff --git a/configure b/configure index 44912b9c34df8..69b12309de578 100755 --- a/configure +++ b/configure @@ -12902,6 +12902,50 @@ if test "x$ac_cv_lib_sqlite3_sqlite3_load_extension" = xyes; then : else have_sqlite3_load_extension=no +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_serialize in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_serialize in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_serialize+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_serialize (); +int +main () +{ +return sqlite3_serialize (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_serialize=yes +else + ac_cv_lib_sqlite3_sqlite3_serialize=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_serialize" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_serialize" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_serialize" = xyes; then : + + +$as_echo "#define PY_SQLITE_HAVE_SERIALIZE 1" >>confdefs.h + + fi diff --git a/configure.ac b/configure.ac index c02adf7bf3f14..5860595b752c8 100644 --- a/configure.ac +++ b/configure.ac @@ -3605,6 +3605,12 @@ dnl hence CPPFLAGS instead of CFLAGS. [have_sqlite3_load_extension=yes], [have_sqlite3_load_extension=no] ) + AC_CHECK_LIB([sqlite3], [sqlite3_serialize], [ + AC_DEFINE( + [PY_SQLITE_HAVE_SERIALIZE], [1], + [Define if SQLite was compiled with the serialize API] + ) + ]) ], [ have_supported_sqlite3=no ]) diff --git a/pyconfig.h.in b/pyconfig.h.in index be776f734163f..4ac054a28d8ef 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1506,6 +1506,9 @@ /* Define to 1 to build the sqlite module with loadable extensions support. */ #undef PY_SQLITE_ENABLE_LOAD_EXTENSION +/* Define if SQLite was compiled with the serialize API */ +#undef PY_SQLITE_HAVE_SERIALIZE + /* Default cipher suites list for ssl module. 1: Python's preferred selection, 2: leave OpenSSL defaults untouched, 0: custom string */ #undef PY_SSL_DEFAULT_CIPHERS [View Less]

1 0

bpo-47212: Improve error messages for un-parenthesized generator expressions (GH-32302)
by pablogsal April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/aa0f056a00c4bcaef83d729e042359ddae… commit: aa0f056a00c4bcaef83d729e042359ddae903382 branch: main author: Matthieu Dartiailh <marul(a)laposte.net> committer: pablogsal <Pablogsal(a)gmail.com> date: 2022-04-05T14:47:13+01:00 summary: bpo-47212: Improve error messages for un-parenthesized generator expressions (GH-32302) files: A Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst M Grammar/python.gram M Lib/test/… [View More]test_exceptions.py M Lib/test/test_syntax.py M Parser/action_helpers.c M Parser/parser.c M Parser/pegen.h diff --git a/Grammar/python.gram b/Grammar/python.gram index b9965d224a85b..15c40b6bbbacd 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1073,12 +1073,12 @@ func_type_comment[Token*]: invalid_arguments: | a=args ',' '*' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable argument unpacking follows keyword argument unpacking") } | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, comprehension_ty)->target, "Generator expression must be parenthesized") } + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")} | a=args b=for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a, b) } | args ',' a=expression b=for_if_clauses { - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") } + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } invalid_kwarg: | a[Token*]=('True'|'False'|'None') b='=' { @@ -1257,7 +1257,7 @@ invalid_finally_stmt: invalid_except_stmt_indent: | a='except' expression ['as' NAME ] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) } - | a='except' ':' NEWLINE !INDENT { RAISE_SYNTAX_ERROR("expected an indented block after except statement on line %d", a->lineno) } + | a='except' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) } invalid_except_star_stmt_indent: | a='except' '*' expression ['as' NAME ] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'except*' statement on line %d", a->lineno) } diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index a75b7fae5518e..6dca79efef180 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -198,12 +198,17 @@ def ckmsg(src, msg, exception=SyntaxError): s = '''if True:\n print()\n\texec "mixed tabs and spaces"''' ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError) - def check(self, src, lineno, offset, encoding='utf-8'): + def check(self, src, lineno, offset, end_lineno=None, end_offset=None, encoding='utf-8'): with self.subTest(source=src, lineno=lineno, offset=offset): with self.assertRaises(SyntaxError) as cm: compile(src, '<fragment>', 'exec') self.assertEqual(cm.exception.lineno, lineno) self.assertEqual(cm.exception.offset, offset) + if end_lineno is not None: + self.assertEqual(cm.exception.end_lineno, end_lineno) + if end_offset is not None: + self.assertEqual(cm.exception.end_offset, end_offset) + if cm.exception.text is not None: if not isinstance(src, str): src = src.decode(encoding, 'replace') @@ -235,6 +240,10 @@ def testSyntaxErrorOffset(self): check('match ...:\n case {**rest, "key": value}:\n ...', 2, 19) check("[a b c d e f]", 1, 2) check("for x yfff:", 1, 7) + check("f(a for a in b, c)", 1, 3, 1, 15) + check("f(a for a in b if a, c)", 1, 3, 1, 20) + check("f(a, b for b in c)", 1, 6, 1, 18) + check("f(a, b for b in c, d)", 1, 6, 1, 18) # Errors thrown by compile.c check('class foo:return 1', 1, 11) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 3e79ebfc68351..96e5c129c6599 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1307,6 +1307,13 @@ Traceback (most recent call last): IndentationError: expected an indented block after 'try' statement on line 1 + >>> try: + ... something() + ... except: + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after 'except' statement on line 3 + >>> try: ... something() ... except A: diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst b/Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst new file mode 100644 index 0000000000000..8f1f6b6cfbbb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-04-05-11-29-21.bpo-47212.leF4pz.rst @@ -0,0 +1,3 @@ +Raise :exc:`IndentationError` instead of :exc:`SyntaxError` for a bare +``except`` with no following indent. Improve :exc:`SyntaxError` locations for +an un-parenthesized generator used as arguments. Patch by Matthieu Dartiailh. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index e5d7b667f7f5e..d1be679aff2e7 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1145,7 +1145,7 @@ _PyPegen_get_expr_name(expr_ty e) } } -static inline expr_ty +expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension) { if (comprehension->ifs == NULL || asdl_seq_LEN(comprehension->ifs) == 0) { return comprehension->iter; diff --git a/Parser/parser.c b/Parser/parser.c index 40c5d62fe49d7..adc8d509eb7d7 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -18968,7 +18968,7 @@ invalid_arguments_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); - _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , PyPegen_last_item ( b , comprehension_ty ) -> target , "Generator expression must be parenthesized" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , _PyPegen_get_last_comprehension_item ( PyPegen_last_item ( b , comprehension_ty ) ) , "Generator expression must be parenthesized" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -19061,7 +19061,7 @@ invalid_arguments_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ',' expression for_if_clauses")); - _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , asdl_seq_GET ( b , b -> size - 1 ) -> target , "Generator expression must be parenthesized" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , _PyPegen_get_last_comprehension_item ( PyPegen_last_item ( b , comprehension_ty ) ) , "Generator expression must be parenthesized" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22190,7 +22190,7 @@ invalid_except_stmt_indent_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_except_stmt_indent[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' ':' NEWLINE !INDENT")); - _res = RAISE_SYNTAX_ERROR ( "expected an indented block after except statement on line %d" , a -> lineno ); + _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'except' statement on line %d" , a -> lineno ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index 77d5ca8418a97..fe0c327b87556 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -324,6 +324,7 @@ int _PyPegen_check_barry_as_flufl(Parser *, Token *); int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t); mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *); void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); +expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension); void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions); // Parser API [View Less]

1 0

bpo-45774: Harden SQLite detection (GH-30016)
by tiran April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/f1606a5ba50bdc4e7d335d62297b4b4043… commit: f1606a5ba50bdc4e7d335d62297b4b4043a25e6e branch: main author: Erlend Egeberg Aasland <erlend.aasland(a)innova.no> committer: tiran <christian(a)python.org> date: 2022-04-05T14:54:47+02:00 summary: bpo-45774: Harden SQLite detection (GH-30016) files: A Misc/NEWS.d/next/Build/2022-03-04-21-24-02.bpo-45774.9AhC0r.rst M configure M configure.ac M pyconfig.h.in diff --git a/Misc/NEWS.d/next/Build/2022-… [View More]03-04-21-24-02.bpo-45774.9AhC0r.rst b/Misc/NEWS.d/next/Build/2022-03-04-21-24-02.bpo-45774.9AhC0r.rst new file mode 100644 index 0000000000000..0bfc9862ef5c7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-03-04-21-24-02.bpo-45774.9AhC0r.rst @@ -0,0 +1,2 @@ +``configure`` now verifies that all SQLite C APIs needed for the +:mod:`sqlite3` extension module are found. diff --git a/configure b/configure index d6bc7175c2fa0..44912b9c34df8 100755 --- a/configure +++ b/configure @@ -1742,8 +1742,8 @@ Optional Features: --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-loadable-sqlite-extensions - support loadable extensions in _sqlite module, see - Doc/library/sqlite3.rst (default is no) + support loadable extensions in the sqlite3 module, + see Doc/library/sqlite3.rst (default is no) --enable-ipv6 enable ipv6 (with ipv4) support, see Doc/library/socket.rst (default is yes if supported) --enable-big-digits[=15|30] @@ -12262,20 +12262,47 @@ $as_echo "yes" >&6; } fi as_fn_append LIBSQLITE3_CFLAGS ' -I$(srcdir)/Modules/_sqlite' + + save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS + CPPFLAGS="$LIBSQLITE3_CFLAGS $CFLAGS" LDFLAGS="$LIBSQLITE3_LIBS $LDFLAGS" ac_fn_c_check_header_mongrel "$LINENO" "sqlite3.h" "ac_cv_header_sqlite3_h" "$ac_includes_default" if test "x$ac_cv_header_sqlite3_h" = xyes; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_open_v2 in -lsqlite3" >&5 -$as_echo_n "checking for sqlite3_open_v2 in -lsqlite3... " >&6; } -if ${ac_cv_lib_sqlite3_sqlite3_open_v2+:} false; then : + have_sqlite3=yes + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include <sqlite3.h> + #if SQLITE_VERSION_NUMBER < 3007015 + # error "SQLite 3.7.15 or higher required" + #endif + +int +main () +{ + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + + have_supported_sqlite3=yes + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_bind_double in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_bind_double in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_bind_double+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -12289,59 +12316,552 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char sqlite3_open_v2 (); +char sqlite3_bind_double (); int main () { -return sqlite3_open_v2 (); +return sqlite3_bind_double (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_sqlite3_sqlite3_open_v2=yes + ac_cv_lib_sqlite3_sqlite3_bind_double=yes else - ac_cv_lib_sqlite3_sqlite3_open_v2=no + ac_cv_lib_sqlite3_sqlite3_bind_double=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_open_v2" >&5 -$as_echo "$ac_cv_lib_sqlite3_sqlite3_open_v2" >&6; } -if test "x$ac_cv_lib_sqlite3_sqlite3_open_v2" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_bind_double" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_bind_double" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_bind_double" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF - have_sqlite3=yes - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_column_decltype in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_column_decltype in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_column_decltype+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_column_decltype (); +int +main () +{ +return sqlite3_column_decltype (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_column_decltype=yes +else + ac_cv_lib_sqlite3_sqlite3_column_decltype=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_column_decltype" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_column_decltype" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_column_decltype" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi - #include <sqlite3.h> - #if SQLITE_VERSION_NUMBER < 3007015 - # error "SQLite 3.7.15 or higher required" - #endif + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_column_double in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_column_double in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_column_double+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_column_double (); int main () { +return sqlite3_column_double (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_column_double=yes +else + ac_cv_lib_sqlite3_sqlite3_column_double=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_column_double" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_column_double" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_column_double" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_complete in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_complete in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_complete+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_complete (); +int +main () +{ +return sqlite3_complete (); ; return 0; } +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_complete=yes +else + ac_cv_lib_sqlite3_sqlite3_complete=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_complete" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_complete" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_complete" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no +fi + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_enable_shared_cache in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_enable_shared_cache in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_enable_shared_cache+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_enable_shared_cache (); +int +main () +{ +return sqlite3_enable_shared_cache (); + ; + return 0; +} _ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - have_supported_sqlite3=yes +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_enable_shared_cache=yes else - have_supported_sqlite3=no + ac_cv_lib_sqlite3_sqlite3_enable_shared_cache=no fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_enable_shared_cache" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_enable_shared_cache" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_enable_shared_cache" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_progress_handler in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_progress_handler in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_progress_handler+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_progress_handler (); +int +main () +{ +return sqlite3_progress_handler (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_progress_handler=yes +else + ac_cv_lib_sqlite3_sqlite3_progress_handler=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_progress_handler" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_progress_handler" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_progress_handler" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_result_double in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_result_double in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_result_double+:} false; then : + $as_echo_n "(cached) " >&6 else - have_sqlite3=no + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_result_double (); +int +main () +{ +return sqlite3_result_double (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_result_double=yes +else + ac_cv_lib_sqlite3_sqlite3_result_double=no fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_result_double" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_result_double" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_result_double" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_load_extension in -lsqlite3" >&5 + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_set_authorizer in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_set_authorizer in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_set_authorizer+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_set_authorizer (); +int +main () +{ +return sqlite3_set_authorizer (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_set_authorizer=yes +else + ac_cv_lib_sqlite3_sqlite3_set_authorizer=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_set_authorizer" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_set_authorizer" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_set_authorizer" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_trace_v2 in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_trace_v2 in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_trace_v2+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_trace_v2 (); +int +main () +{ +return sqlite3_trace_v2 (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_trace_v2=yes +else + ac_cv_lib_sqlite3_sqlite3_trace_v2=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_trace_v2" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_trace_v2" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_trace_v2" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_trace in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_trace in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_trace+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_trace (); +int +main () +{ +return sqlite3_trace (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_trace=yes +else + ac_cv_lib_sqlite3_sqlite3_trace=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_trace" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_trace" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_trace" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + + +fi + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_value_double in -lsqlite3" >&5 +$as_echo_n "checking for sqlite3_value_double in -lsqlite3... " >&6; } +if ${ac_cv_lib_sqlite3_sqlite3_value_double+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_value_double (); +int +main () +{ +return sqlite3_value_double (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_sqlite3_sqlite3_value_double=yes +else + ac_cv_lib_sqlite3_sqlite3_value_double=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_value_double" >&5 +$as_echo "$ac_cv_lib_sqlite3_sqlite3_value_double" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_value_double" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSQLITE3 1 +_ACEOF + + LIBS="-lsqlite3 $LIBS" + +else + + have_supported_sqlite3=no + +fi + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_load_extension in -lsqlite3" >&5 $as_echo_n "checking for sqlite3_load_extension in -lsqlite3... " >&6; } if ${ac_cv_lib_sqlite3_sqlite3_load_extension+:} false; then : $as_echo_n "(cached) " >&6 @@ -12381,9 +12901,17 @@ if test "x$ac_cv_lib_sqlite3_sqlite3_load_extension" = xyes; then : have_sqlite3_load_extension=yes else have_sqlite3_load_extension=no + fi +else + + have_supported_sqlite3=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + fi @@ -12395,30 +12923,37 @@ LIBS=$save_LIBS -# Check for support for loadable sqlite extensions { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-loadable-sqlite-extensions" >&5 $as_echo_n "checking for --enable-loadable-sqlite-extensions... " >&6; } # Check whether --enable-loadable-sqlite-extensions was given. if test "${enable_loadable_sqlite_extensions+set}" = set; then : - enableval=$enable_loadable_sqlite_extensions; if test "x$have_sqlite3_load_extension" = xno; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Your version of SQLite does not support loadable extensions" >&5 + enableval=$enable_loadable_sqlite_extensions; + if test "x$have_sqlite3_load_extension" = xno; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: n/a" >&5 +$as_echo "n/a" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Your version of SQLite does not support loadable extensions" >&5 $as_echo "$as_me: WARNING: Your version of SQLite does not support loadable extensions" >&2;} -fi + else - enable_loadable_sqlite_extensions=no -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_loadable_sqlite_extensions" >&5 -$as_echo "$enable_loadable_sqlite_extensions" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } -if test "x$enable_loadable_sqlite_extensions" = xyes; then : +$as_echo "#define PY_SQLITE_ENABLE_LOAD_EXTENSION 1" >>confdefs.h -$as_echo "#define PY_SQLITE_ENABLE_LOAD_EXTENSION 1" >>confdefs.h +fi + +else + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } fi + found_tcltk=no for _QUERY in \ "tcl >= 8.5.12 tk >= 8.5.12" \ diff --git a/configure.ac b/configure.ac index 53bbc3e7b199c..c02adf7bf3f14 100644 --- a/configure.ac +++ b/configure.ac @@ -3562,45 +3562,79 @@ PKG_CHECK_MODULES( ) AS_VAR_APPEND([LIBSQLITE3_CFLAGS], [' -I$(srcdir)/Modules/_sqlite']) -WITH_SAVE_ENV( +dnl PY_CHECK_SQLITE_FUNC(FUNCTION, IF-FOUND, IF-NOT-FOUND) +AC_DEFUN([PY_CHECK_SQLITE_FUNC], [ + AC_CHECK_LIB([sqlite3], [$1], [$2], [ + m4_ifblank([$3], [have_supported_sqlite3=no], [$3]) + ]) +]) + +WITH_SAVE_ENV([ dnl bpo-45774/GH-29507: The CPP check in AC_CHECK_HEADER can fail on FreeBSD, dnl hence CPPFLAGS instead of CFLAGS. CPPFLAGS="$LIBSQLITE3_CFLAGS $CFLAGS" LDFLAGS="$LIBSQLITE3_LIBS $LDFLAGS" AC_CHECK_HEADER([sqlite3.h], [ - AC_CHECK_LIB([sqlite3], [sqlite3_open_v2], [ - have_sqlite3=yes - AC_COMPILE_IFELSE([ - AC_LANG_PROGRAM([ - #include <sqlite3.h> - #if SQLITE_VERSION_NUMBER < 3007015 - # error "SQLite 3.7.15 or higher required" - #endif - ], []) - ], [have_supported_sqlite3=yes], [have_supported_sqlite3=no]) - ], [have_sqlite3=no]) - AC_CHECK_LIB([sqlite3], [sqlite3_load_extension], - [have_sqlite3_load_extension=yes], - [have_sqlite3_load_extension=no]) - ]) -) + have_sqlite3=yes -# Check for support for loadable sqlite extensions -AC_MSG_CHECKING(for --enable-loadable-sqlite-extensions) -AC_ARG_ENABLE(loadable-sqlite-extensions, - AS_HELP_STRING([--enable-loadable-sqlite-extensions], - [support loadable extensions in _sqlite module, see Doc/library/sqlite3.rst (default is no)]), - [AS_VAR_IF([have_sqlite3_load_extension], [no], - [AC_MSG_WARN([Your version of SQLite does not support loadable extensions])])], - [enable_loadable_sqlite_extensions=no]) -AC_MSG_RESULT($enable_loadable_sqlite_extensions) - -AS_VAR_IF([enable_loadable_sqlite_extensions], [yes], [ - AC_DEFINE(PY_SQLITE_ENABLE_LOAD_EXTENSION, 1, - [Define to 1 to build the sqlite module with loadable extensions support.]) + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([ + #include <sqlite3.h> + #if SQLITE_VERSION_NUMBER < 3007015 + # error "SQLite 3.7.15 or higher required" + #endif + ], []) + ], [ + have_supported_sqlite3=yes + dnl Check that required functions are in place. A lot of stuff may be + dnl omitted with SQLITE_OMIT_* compile time defines. + PY_CHECK_SQLITE_FUNC([sqlite3_bind_double]) + PY_CHECK_SQLITE_FUNC([sqlite3_column_decltype]) + PY_CHECK_SQLITE_FUNC([sqlite3_column_double]) + PY_CHECK_SQLITE_FUNC([sqlite3_complete]) + PY_CHECK_SQLITE_FUNC([sqlite3_enable_shared_cache]) + PY_CHECK_SQLITE_FUNC([sqlite3_progress_handler]) + PY_CHECK_SQLITE_FUNC([sqlite3_result_double]) + PY_CHECK_SQLITE_FUNC([sqlite3_set_authorizer]) + PY_CHECK_SQLITE_FUNC([sqlite3_trace_v2], [], [ + PY_CHECK_SQLITE_FUNC([sqlite3_trace]) + ]) + PY_CHECK_SQLITE_FUNC([sqlite3_value_double]) + AC_CHECK_LIB([sqlite3], [sqlite3_load_extension], + [have_sqlite3_load_extension=yes], + [have_sqlite3_load_extension=no] + ) + ], [ + have_supported_sqlite3=no + ]) + ]) ]) +dnl Check for support for loadable sqlite extensions +AC_MSG_CHECKING([for --enable-loadable-sqlite-extensions]) +AC_ARG_ENABLE([loadable-sqlite-extensions], + AS_HELP_STRING( + [--enable-loadable-sqlite-extensions], [ + support loadable extensions in the sqlite3 module, see + Doc/library/sqlite3.rst (default is no) + ] + ), [ + AS_VAR_IF([have_sqlite3_load_extension], [no], [ + AC_MSG_RESULT([n/a]) + AC_MSG_WARN([Your version of SQLite does not support loadable extensions]) + ], [ + AC_MSG_RESULT([yes]) + AC_DEFINE( + [PY_SQLITE_ENABLE_LOAD_EXTENSION], [1], + [Define to 1 to build the sqlite module with loadable extensions support.] + ) + ]) + ], [ + AC_MSG_RESULT([no]) + ] +) + dnl dnl Detect Tcl/Tk. Use pkg-config if available. dnl diff --git a/pyconfig.h.in b/pyconfig.h.in index 3d2020c896934..be776f734163f 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -655,6 +655,9 @@ /* Define to 1 if you have the `sendfile' library (-lsendfile). */ #undef HAVE_LIBSENDFILE +/* Define to 1 if you have the `sqlite3' library (-lsqlite3). */ +#undef HAVE_LIBSQLITE3 + /* Define to 1 if you have the <libutil.h> header file. */ #undef HAVE_LIBUTIL_H [View Less]

1 0

bpo-47120: make JUMP_NO_INTERRUPT relative (GH-32221)
by iritkatriel April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/0aa8d5cbd89cf3b61d7e8626f3a7b9c488… commit: 0aa8d5cbd89cf3b61d7e8626f3a7b9c4881dfd70 branch: main author: Irit Katriel <1055913+iritkatriel(a)users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel(a)users.noreply.github.com> date: 2022-04-05T12:49:08+01:00 summary: bpo-47120: make JUMP_NO_INTERRUPT relative (GH-32221) files: A Misc/NEWS.d/next/Core and Builtins/2022-03-31-21-43-57.bpo-47120.NgxQbA.rst M Doc/library/dis.rst M … [View More]Doc/whatsnew/3.11.rst M Include/opcode.h M Lib/importlib/_bootstrap_external.py M Lib/opcode.py M Lib/test/test_dis.py M Objects/frameobject.c M Python/ceval.c M Python/compile.c M Python/opcode_targets.h diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 63d846715b165..fa0e23a6c9693 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -925,7 +925,14 @@ iterations of the loop. .. opcode:: JUMP_BACKWARD (delta) - Decrements bytecode counter by *delta*. + Decrements bytecode counter by *delta*. Checks for interrupts. + + .. versionadded:: 3.11 + + +.. opcode:: JUMP_BACKWARD_NO_INTERRUPT (delta) + + Decrements bytecode counter by *delta*. Does not check for interrupts. .. versionadded:: 3.11 @@ -974,13 +981,6 @@ iterations of the loop. .. versionadded:: 3.1 -.. opcode:: JUMP_NO_INTERRUPT (target) - - Set bytecode counter to *target*. Do not check for interrupts. - - .. versionadded:: 3.11 - - .. opcode:: FOR_ITER (delta) TOS is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index fae85d867f91b..d0c10a9100997 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -528,6 +528,8 @@ CPython bytecode changes * Replaced :opcode:`JUMP_ABSOLUTE` by the relative :opcode:`JUMP_BACKWARD`. +* Added :opcode:`JUMP_BACKWARD_NO_INTERRUPT`, which is used in certain loops where it is undesirable to handle interrupts. + Deprecated ========== diff --git a/Include/opcode.h b/Include/opcode.h index fd49dfed27ecd..ff3ffddda2147 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -87,7 +87,7 @@ extern "C" { #define GET_AWAITABLE 131 #define MAKE_FUNCTION 132 #define BUILD_SLICE 133 -#define JUMP_NO_INTERRUPT 134 +#define JUMP_BACKWARD_NO_INTERRUPT 134 #define MAKE_CELL 135 #define LOAD_CLOSURE 136 #define LOAD_DEREF 137 @@ -196,7 +196,7 @@ static const uint32_t _PyOpcode_RelativeJump[8] = { 0U, 536870912U, 134234112U, - 4096U, + 4160U, 0U, 0U, 0U, @@ -292,11 +292,11 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_STAR] = IMPORT_STAR, [IS_OP] = IS_OP, [JUMP_BACKWARD] = JUMP_BACKWARD, + [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, [JUMP_BACKWARD_QUICK] = JUMP_BACKWARD, [JUMP_FORWARD] = JUMP_FORWARD, [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, - [JUMP_NO_INTERRUPT] = JUMP_NO_INTERRUPT, [KW_NAMES] = KW_NAMES, [LIST_APPEND] = LIST_APPEND, [LIST_EXTEND] = LIST_EXTEND, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 189547cc16a8c..45be177df76a9 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -398,7 +398,8 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) # Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) # Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH) +# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, +# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) # Python 3.12 will start with magic number 3500 diff --git a/Lib/opcode.py b/Lib/opcode.py index 23d98df5061f4..97b580532cb94 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -154,7 +154,7 @@ def jabs_op(name, op, entries=0): def_op('GET_AWAITABLE', 131) def_op('MAKE_FUNCTION', 132) # Flags def_op('BUILD_SLICE', 133) # Number of items -jabs_op('JUMP_NO_INTERRUPT', 134) # Target byte offset from beginning of code +jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) def_op('MAKE_CELL', 135) hasfree.append(135) def_op('LOAD_CLOSURE', 136) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 544e1350cb7f9..2f78d42cc724a 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -684,7 +684,8 @@ def test_boundaries(self): def test_widths(self): for opcode, opname in enumerate(dis.opname): if opname in ('BUILD_MAP_UNPACK_WITH_CALL', - 'BUILD_TUPLE_UNPACK_WITH_CALL'): + 'BUILD_TUPLE_UNPACK_WITH_CALL', + 'JUMP_BACKWARD_NO_INTERRUPT'): continue with self.subTest(opname=opname): width = dis._OPNAME_WIDTH diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-31-21-43-57.bpo-47120.NgxQbA.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-31-21-43-57.bpo-47120.NgxQbA.rst new file mode 100644 index 0000000000000..236ad94795056 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-31-21-43-57.bpo-47120.NgxQbA.rst @@ -0,0 +1 @@ +Replace the absolute jump opcode :opcode:`JUMP_NO_INTERRUPT` by the relative :opcode:`JUMP_BACKWARD_NO_INTERRUPT`. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c257c0a3b43f2..6842e62839fd1 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -229,15 +229,6 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[i+1] = next_stack; break; } - case JUMP_NO_INTERRUPT: - j = get_arg(code, i); - assert(j < len); - if (stacks[j] == UNINITIALIZED && j < i) { - todo = 1; - } - assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); - stacks[j] = next_stack; - break; case POP_EXCEPT: next_stack = pop_value(pop_value(pop_value(next_stack))); stacks[i+1] = next_stack; @@ -256,8 +247,13 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[j] = next_stack; break; case JUMP_BACKWARD: + case JUMP_BACKWARD_NO_INTERRUPT: j = i + 1 - get_arg(code, i); assert(j >= 0); + assert(j < len); + if (stacks[j] == UNINITIALIZED && j < i) { + todo = 1; + } assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; break; diff --git a/Python/ceval.c b/Python/ceval.c index f7e08c6ee813a..9b7c42cbe4b11 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4045,13 +4045,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); } - TARGET(JUMP_NO_INTERRUPT) { + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ - JUMPTO(oparg); + JUMPBY(-oparg); DISPATCH(); } diff --git a/Python/compile.c b/Python/compile.c index c92b267dd03a1..f04ba9ec50f6f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -77,8 +77,9 @@ #define SETUP_WITH -3 #define POP_BLOCK -4 #define JUMP -5 +#define JUMP_NO_INTERRUPT -6 -#define MIN_VIRTUAL_OPCODE -5 +#define MIN_VIRTUAL_OPCODE -6 #define MAX_ALLOWED_OPCODE 254 #define IS_WITHIN_OPCODE_RANGE(opcode) \ @@ -86,6 +87,13 @@ #define IS_VIRTUAL_OPCODE(opcode) ((opcode) < 0) +/* opcodes which are not emitted in codegen stage, only by the assembler */ +#define IS_ASSEMBLER_OPCODE(opcode) \ + ((opcode) == JUMP_FORWARD || \ + (opcode) == JUMP_BACKWARD || \ + (opcode) == JUMP_BACKWARD_NO_INTERRUPT) + + #define IS_TOP_LEVEL_AWAIT(c) ( \ (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ && (c->u->u_ste->ste_type == ModuleBlock)) @@ -1011,6 +1019,7 @@ stack_effect(int opcode, int oparg, int jump) case JUMP_FORWARD: case JUMP_BACKWARD: case JUMP: + case JUMP_BACKWARD_NO_INTERRUPT: case JUMP_NO_INTERRUPT: return 0; @@ -1199,6 +1208,7 @@ compiler_addop_line(struct compiler *c, int opcode, int line, int end_line, int col_offset, int end_col_offset) { assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); assert(!HAS_ARG(opcode) || IS_ARTIFICIAL(opcode)); if (compiler_use_new_implicit_block_if_needed(c) < 0) { @@ -1442,6 +1452,7 @@ compiler_addop_i_line(struct compiler *c, int opcode, Py_ssize_t oparg, EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); assert(HAS_ARG(opcode)); assert(0 <= oparg && oparg <= 2147483647); @@ -1486,6 +1497,7 @@ static int add_jump_to_block(struct compiler *c, int opcode, basicblock *target) { assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); assert(HAS_ARG(opcode) || IS_VIRTUAL_OPCODE(opcode)); assert(target != NULL); @@ -7089,8 +7101,7 @@ stackdepth(struct compiler *c) stackdepth_push(&sp, instr->i_target, target_depth); } depth = new_depth; - assert(instr->i_opcode != JUMP_FORWARD); - assert(instr->i_opcode != JUMP_BACKWARD); + assert(!IS_ASSEMBLER_OPCODE(instr->i_opcode)); if (instr->i_opcode == JUMP_NO_INTERRUPT || instr->i_opcode == JUMP || instr->i_opcode == RETURN_VALUE || @@ -7597,15 +7608,15 @@ normalize_jumps(struct assembler *a) continue; } struct instr *last = &b->b_instr[b->b_iused-1]; - assert(last->i_opcode != JUMP_FORWARD); - assert(last->i_opcode != JUMP_BACKWARD); + assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); if (last->i_opcode == JUMP) { - if (last->i_target->b_visited == 0) { - last->i_opcode = JUMP_FORWARD; - } - else { - last->i_opcode = JUMP_BACKWARD; - } + bool is_forward = last->i_target->b_visited == 0; + last->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD; + } + if (last->i_opcode == JUMP_NO_INTERRUPT) { + bool is_forward = last->i_target->b_visited == 0; + last->i_opcode = is_forward ? + JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT; } } } @@ -7641,11 +7652,13 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) instr->i_oparg = instr->i_target->b_offset; if (is_relative_jump(instr)) { if (instr->i_oparg < bsize) { - assert(instr->i_opcode == JUMP_BACKWARD); + assert(instr->i_opcode == JUMP_BACKWARD || + instr->i_opcode == JUMP_BACKWARD_NO_INTERRUPT); instr->i_oparg = bsize - instr->i_oparg; } else { assert(instr->i_opcode != JUMP_BACKWARD); + assert(instr->i_opcode != JUMP_BACKWARD_NO_INTERRUPT); instr->i_oparg -= bsize; } } @@ -8667,14 +8680,12 @@ optimize_basic_block(struct compiler *c, basicblock *bb, PyObject *consts) inst->i_target = inst->i_target->b_next; } target = &inst->i_target->b_instr[0]; - assert(target->i_opcode != JUMP_FORWARD); - assert(target->i_opcode != JUMP_BACKWARD); + assert(!IS_ASSEMBLER_OPCODE(target->i_opcode)); } else { target = &nop; } - assert(inst->i_opcode != JUMP_FORWARD); - assert(inst->i_opcode != JUMP_BACKWARD); + assert(!IS_ASSEMBLER_OPCODE(inst->i_opcode)); switch (inst->i_opcode) { /* Remove LOAD_CONST const; conditional jump */ case LOAD_CONST: @@ -8975,8 +8986,7 @@ normalize_basic_block(basicblock *bb) { /* Mark blocks as exit and/or nofallthrough. Raise SystemError if CFG is malformed. */ for (int i = 0; i < bb->b_iused; i++) { - assert(bb->b_instr[i].i_opcode != JUMP_FORWARD); - assert(bb->b_instr[i].i_opcode != JUMP_BACKWARD); + assert(!IS_ASSEMBLER_OPCODE(bb->b_instr[i].i_opcode)); switch(bb->b_instr[i].i_opcode) { case RETURN_VALUE: case RAISE_VARARGS: @@ -9163,8 +9173,7 @@ optimize_cfg(struct compiler *c, struct assembler *a, PyObject *consts) for (basicblock *b = a->a_entry; b != NULL; b = b->b_next) { if (b->b_iused > 0) { struct instr *b_last_instr = &b->b_instr[b->b_iused - 1]; - assert(b_last_instr->i_opcode != JUMP_FORWARD); - assert(b_last_instr->i_opcode != JUMP_BACKWARD); + assert(!IS_ASSEMBLER_OPCODE(b_last_instr->i_opcode)); if (b_last_instr->i_opcode == JUMP || b_last_instr->i_opcode == JUMP_NO_INTERRUPT) { if (b_last_instr->i_target == b->b_next) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index e71b3e2120f7a..064aa060c8428 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -133,7 +133,7 @@ static void *opcode_targets[256] = { &&TARGET_GET_AWAITABLE, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, - &&TARGET_JUMP_NO_INTERRUPT, + &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, &&TARGET_MAKE_CELL, &&TARGET_LOAD_CLOSURE, &&TARGET_LOAD_DEREF, [View Less]

1 0

bpo-47186: Replace JUMP_IF_NOT_EG_MATCH by CHECK_EG_MATCH + jump (GH-32309)
by iritkatriel April 5, 2022

April 5, 2022

https://github.com/python/cpython/commit/32091df41ce6e3a71df2cf37dc74b728c0… commit: 32091df41ce6e3a71df2cf37dc74b728c0d885f2 branch: main author: Irit Katriel <1055913+iritkatriel(a)users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel(a)users.noreply.github.com> date: 2022-04-05T12:06:22+01:00 summary: bpo-47186: Replace JUMP_IF_NOT_EG_MATCH by CHECK_EG_MATCH + jump (GH-32309) files: A Misc/NEWS.d/next/Core and Builtins/2022-04-04-17-41-10.bpo-47186.aQWoSh.rst M … [View More]Doc/library/dis.rst M Doc/whatsnew/3.11.rst M Include/opcode.h M Lib/importlib/_bootstrap_external.py M Lib/opcode.py M Objects/frameobject.c M Python/ceval.c M Python/compile.c M Python/opcode_targets.h diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index b364e3f031b3d..63d846715b165 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -635,6 +635,28 @@ iterations of the loop. .. versionadded:: 3.11 +.. opcode:: CHECK_EG_MATCH + + Performs exception matching for ``except*``. Applies ``split(TOS)`` on + the exception group representing TOS1. + + In case of a match, pops two items from the stack and pushes the + non-matching subgroup (``None`` in case of full match) followed by the + matching subgroup. When there is no match, pops one item (the match + type) and pushes ``None``. + + .. versionadded:: 3.11 + +.. opcode:: PREP_RERAISE_STAR + + Combines the raised and reraised exceptions list from TOS, into an exception + group to propagate from a try-except* block. Uses the original exception + group from TOS1 to reconstruct the structure of reraised exceptions. Pops + two items from the stack and pushes the exception to reraise or ``None`` + if there isn't one. + + .. versionadded:: 3.11 + .. opcode:: WITH_EXCEPT_START Calls the function in position 4 on the stack with arguments (type, val, tb) @@ -922,18 +944,6 @@ iterations of the loop. .. versionadded:: 3.1 -.. opcode:: JUMP_IF_NOT_EG_MATCH (target) - - Performs exception matching for ``except*``. Applies ``split(TOS)`` on - the exception group representing TOS1. Jumps if no match is found. - - Pops one item from the stack (the match type). If a match was found, - next item (the exception) and pushes the non-matching part of the - exception group followed by the matching part. - - .. versionadded:: 3.11 - - .. opcode:: POP_JUMP_IF_NOT_NONE (target) If TOS is not none, sets the bytecode counter to *target*. TOS is popped. @@ -948,17 +958,6 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: PREP_RERAISE_STAR - - Combines the raised and reraised exceptions list from TOS, into an exception - group to propagate from a try-except* block. Uses the original exception - group from TOS1 to reconstruct the structure of reraised exceptions. Pops - two items from the stack and pushes the exception to reraise or ``None`` - if there isn't one. - - .. versionadded:: 3.11 - - .. opcode:: JUMP_IF_TRUE_OR_POP (target) If TOS is true, sets the bytecode counter to *target* and leaves TOS on the diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index c607aadb4457c..fae85d867f91b 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -523,6 +523,9 @@ CPython bytecode changes * Replaced :opcode:`JUMP_IF_NOT_EXC_MATCH` by :opcode:`CHECK_EXC_MATCH` which performs the check but does not jump. +* Replaced :opcode:`JUMP_IF_NOT_EG_MATCH` by :opcode:`CHECK_EG_MATCH` which + performs the check but does not jump. + * Replaced :opcode:`JUMP_ABSOLUTE` by the relative :opcode:`JUMP_BACKWARD`. Deprecated diff --git a/Include/opcode.h b/Include/opcode.h index c82d1fd1fae2b..fd49dfed27ecd 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -22,6 +22,7 @@ extern "C" { #define MATCH_KEYS 33 #define PUSH_EXC_INFO 35 #define CHECK_EXC_MATCH 36 +#define CHECK_EG_MATCH 37 #define WITH_EXCEPT_START 49 #define GET_AITER 50 #define GET_ANEXT 51 @@ -80,7 +81,6 @@ extern "C" { #define LOAD_FAST 124 #define STORE_FAST 125 #define DELETE_FAST 126 -#define JUMP_IF_NOT_EG_MATCH 127 #define POP_JUMP_IF_NOT_NONE 128 #define POP_JUMP_IF_NONE 129 #define RAISE_VARARGS 130 @@ -136,39 +136,39 @@ extern "C" { #define COMPARE_OP_INT_JUMP 28 #define COMPARE_OP_STR_JUMP 29 #define JUMP_BACKWARD_QUICK 34 -#define LOAD_ATTR_ADAPTIVE 37 -#define LOAD_ATTR_INSTANCE_VALUE 38 -#define LOAD_ATTR_MODULE 39 -#define LOAD_ATTR_SLOT 40 -#define LOAD_ATTR_WITH_HINT 41 -#define LOAD_CONST__LOAD_FAST 42 -#define LOAD_FAST__LOAD_CONST 43 -#define LOAD_FAST__LOAD_FAST 44 -#define LOAD_GLOBAL_ADAPTIVE 45 -#define LOAD_GLOBAL_BUILTIN 46 -#define LOAD_GLOBAL_MODULE 47 -#define LOAD_METHOD_ADAPTIVE 48 -#define LOAD_METHOD_CLASS 55 -#define LOAD_METHOD_MODULE 56 -#define LOAD_METHOD_NO_DICT 57 -#define LOAD_METHOD_WITH_DICT 58 -#define LOAD_METHOD_WITH_VALUES 59 -#define PRECALL_ADAPTIVE 62 -#define PRECALL_BOUND_METHOD 63 -#define PRECALL_BUILTIN_CLASS 64 -#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 65 -#define PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 66 -#define PRECALL_NO_KW_BUILTIN_FAST 67 -#define PRECALL_NO_KW_BUILTIN_O 72 -#define PRECALL_NO_KW_ISINSTANCE 73 -#define PRECALL_NO_KW_LEN 76 -#define PRECALL_NO_KW_LIST_APPEND 77 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 78 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 79 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 80 -#define PRECALL_NO_KW_STR_1 81 -#define PRECALL_NO_KW_TUPLE_1 113 -#define PRECALL_NO_KW_TYPE_1 121 +#define LOAD_ATTR_ADAPTIVE 38 +#define LOAD_ATTR_INSTANCE_VALUE 39 +#define LOAD_ATTR_MODULE 40 +#define LOAD_ATTR_SLOT 41 +#define LOAD_ATTR_WITH_HINT 42 +#define LOAD_CONST__LOAD_FAST 43 +#define LOAD_FAST__LOAD_CONST 44 +#define LOAD_FAST__LOAD_FAST 45 +#define LOAD_GLOBAL_ADAPTIVE 46 +#define LOAD_GLOBAL_BUILTIN 47 +#define LOAD_GLOBAL_MODULE 48 +#define LOAD_METHOD_ADAPTIVE 55 +#define LOAD_METHOD_CLASS 56 +#define LOAD_METHOD_MODULE 57 +#define LOAD_METHOD_NO_DICT 58 +#define LOAD_METHOD_WITH_DICT 59 +#define LOAD_METHOD_WITH_VALUES 62 +#define PRECALL_ADAPTIVE 63 +#define PRECALL_BOUND_METHOD 64 +#define PRECALL_BUILTIN_CLASS 65 +#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 66 +#define PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 67 +#define PRECALL_NO_KW_BUILTIN_FAST 72 +#define PRECALL_NO_KW_BUILTIN_O 73 +#define PRECALL_NO_KW_ISINSTANCE 76 +#define PRECALL_NO_KW_LEN 77 +#define PRECALL_NO_KW_LIST_APPEND 78 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 79 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 80 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 81 +#define PRECALL_NO_KW_STR_1 113 +#define PRECALL_NO_KW_TUPLE_1 121 +#define PRECALL_NO_KW_TYPE_1 127 #define PRECALL_PYFUNC 141 #define RESUME_QUICK 143 #define STORE_ATTR_ADAPTIVE 150 @@ -205,7 +205,7 @@ static const uint32_t _PyOpcode_Jump[8] = { 0U, 0U, 536870912U, - 2282602496U, + 135118848U, 4163U, 0U, 0U, @@ -259,6 +259,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_PY_EXACT_ARGS] = CALL, [CALL_PY_WITH_DEFAULTS] = CALL, + [CHECK_EG_MATCH] = CHECK_EG_MATCH, [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, [COMPARE_OP] = COMPARE_OP, [COMPARE_OP_ADAPTIVE] = COMPARE_OP, @@ -294,7 +295,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [JUMP_BACKWARD_QUICK] = JUMP_BACKWARD, [JUMP_FORWARD] = JUMP_FORWARD, [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, - [JUMP_IF_NOT_EG_MATCH] = JUMP_IF_NOT_EG_MATCH, [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, [JUMP_NO_INTERRUPT] = JUMP_NO_INTERRUPT, [KW_NAMES] = KW_NAMES, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 39a348aecef6c..189547cc16a8c 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -398,6 +398,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) # Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) # Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) +# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH) # Python 3.12 will start with magic number 3500 @@ -412,7 +413,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3490).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3491).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index e993a5c1ff16a..23d98df5061f4 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -77,6 +77,7 @@ def jabs_op(name, op, entries=0): def_op('PUSH_EXC_INFO', 35) def_op('CHECK_EXC_MATCH', 36) +def_op('CHECK_EG_MATCH', 37) def_op('WITH_EXCEPT_START', 49) def_op('GET_AITER', 50) @@ -147,7 +148,6 @@ def jabs_op(name, op, entries=0): haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) -jabs_op('JUMP_IF_NOT_EG_MATCH', 127) jabs_op('POP_JUMP_IF_NOT_NONE', 128) jabs_op('POP_JUMP_IF_NONE', 129) def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-04-04-17-41-10.bpo-47186.aQWoSh.rst b/Misc/NEWS.d/next/Core and Builtins/2022-04-04-17-41-10.bpo-47186.aQWoSh.rst new file mode 100644 index 0000000000000..2282c485b29e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-04-04-17-41-10.bpo-47186.aQWoSh.rst @@ -0,0 +1 @@ +Replace :opcode:`JUMP_IF_NOT_EG_MATCH` by :opcode:`CHECK_EG_MATCH` + jump. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index fe374bf0632cd..c257c0a3b43f2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -207,7 +207,6 @@ mark_stacks(PyCodeObject *code_obj, int len) case JUMP_IF_TRUE_OR_POP: case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: - case JUMP_IF_NOT_EG_MATCH: { int64_t target_stack; int j = get_arg(code, i); @@ -215,13 +214,8 @@ mark_stacks(PyCodeObject *code_obj, int len) if (stacks[j] == UNINITIALIZED && j < i) { todo = 1; } - if (opcode == JUMP_IF_NOT_EG_MATCH) - { - next_stack = pop_value(pop_value(next_stack)); - target_stack = next_stack; - } - else if (opcode == JUMP_IF_FALSE_OR_POP || - opcode == JUMP_IF_TRUE_OR_POP) + if (opcode == JUMP_IF_FALSE_OR_POP || + opcode == JUMP_IF_TRUE_OR_POP) { target_stack = next_stack; next_stack = pop_value(next_stack); diff --git a/Python/ceval.c b/Python/ceval.c index ce4abd5da2414..f7e08c6ee813a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3804,7 +3804,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); } - TARGET(JUMP_IF_NOT_EG_MATCH) { + TARGET(CHECK_EG_MATCH) { PyObject *match_type = POP(); if (check_except_star_type_valid(tstate, match_type) < 0) { Py_DECREF(match_type); @@ -3825,15 +3825,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(rest == NULL); goto error; } - if (Py_IsNone(match)) { - Py_DECREF(match); + PUSH(match); Py_XDECREF(rest); - /* no match - jump to target */ - JUMPTO(oparg); } else { - /* Total or partial match - update the stack from * [val] * to @@ -3841,17 +3837,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int * (rest can be Py_None) */ - PyObject *exc = TOP(); - SET_TOP(rest); PUSH(match); - PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); - - Py_DECREF(exc); - + Py_DECREF(exc_value); } - DISPATCH(); } diff --git a/Python/compile.c b/Python/compile.c index c3773a4f48929..c92b267dd03a1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1000,8 +1000,8 @@ stack_effect(int opcode, int oparg, int jump) return -1; case CHECK_EXC_MATCH: return 0; - case JUMP_IF_NOT_EG_MATCH: - return jump > 0 ? -1 : 0; + case CHECK_EG_MATCH: + return 0; case IMPORT_NAME: return -1; case IMPORT_FROM: @@ -3533,14 +3533,18 @@ compiler_try_except(struct compiler *c, stmt_ty s) [] POP_BLOCK [] JUMP L0 - [exc] L1: COPY 1 ) save copy of the original exception + [exc] L1: COPY 1 ) save copy of the original exception [orig, exc] BUILD_LIST ) list for raised/reraised excs ("result") [orig, exc, res] SWAP 2 [orig, res, exc] <evaluate E1> - [orig, res, exc, E1] JUMP_IF_NOT_EG_MATCH L2 + [orig, res, exc, E1] CHECK_EG_MATCH + [orig, red, rest/exc, match?] COPY 1 + [orig, red, rest/exc, match?, match?] POP_JUMP_IF_NOT_NONE H1 + [orig, red, exc, None] POP_TOP + [orig, red, exc] JUMP L2 - [orig, res, rest, match] <assign to V1> (or POP if no V1) + [orig, res, rest, match] H1: <assign to V1> (or POP if no V1) [orig, res, rest] SETUP_FINALLY R1 [orig, res, rest] <code for S1> @@ -3622,6 +3626,10 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) if (except == NULL) { return 0; } + basicblock *handle_match = compiler_new_block(c); + if (handle_match == NULL) { + return 0; + } if (i == 0) { /* Push the original EG into the stack */ /* @@ -3641,9 +3649,15 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) } if (handler->v.ExceptHandler.type) { VISIT(c, expr, handler->v.ExceptHandler.type); - ADDOP_JUMP(c, JUMP_IF_NOT_EG_MATCH, except); + ADDOP(c, CHECK_EG_MATCH); + ADDOP_I(c, COPY, 1); + ADDOP_JUMP(c, POP_JUMP_IF_NOT_NONE, handle_match); + ADDOP(c, POP_TOP); // match + ADDOP_JUMP(c, JUMP, except); } + compiler_use_next_block(c, handle_match); + basicblock *cleanup_end = compiler_new_block(c); if (cleanup_end == NULL) { return 0; @@ -3657,7 +3671,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) compiler_nameop(c, handler->v.ExceptHandler.name, Store); } else { - ADDOP(c, POP_TOP); // exc + ADDOP(c, POP_TOP); // match } /* diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 2eba5531723fb..e71b3e2120f7a 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -36,6 +36,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_BACKWARD_QUICK, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, + &&TARGET_CHECK_EG_MATCH, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, @@ -47,40 +48,39 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_BEFORE_ASYNC_WITH, &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, + &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_LOAD_METHOD_CLASS, &&TARGET_LOAD_METHOD_MODULE, &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_LOAD_METHOD_WITH_DICT, - &&TARGET_LOAD_METHOD_WITH_VALUES, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_LOAD_METHOD_WITH_VALUES, &&TARGET_PRECALL_ADAPTIVE, &&TARGET_PRECALL_BOUND_METHOD, &&TARGET_PRECALL_BUILTIN_CLASS, &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, &&TARGET_PRECALL_NO_KW_BUILTIN_O, - &&TARGET_PRECALL_NO_KW_ISINSTANCE, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_PRECALL_NO_KW_ISINSTANCE, &&TARGET_PRECALL_NO_KW_LEN, &&TARGET_PRECALL_NO_KW_LIST_APPEND, &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_PRECALL_NO_KW_STR_1, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_PRECALL_NO_KW_TUPLE_1, + &&TARGET_PRECALL_NO_KW_STR_1, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,13 +120,13 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_PRECALL_NO_KW_TYPE_1, + &&TARGET_PRECALL_NO_KW_TUPLE_1, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, - &&TARGET_JUMP_IF_NOT_EG_MATCH, + &&TARGET_PRECALL_NO_KW_TYPE_1, &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, [View Less]

1 0