Mailman 3 February 2020 - Python-checkins

bpo-21016: pydoc and trace use sysconfig (GH-18476)
by Victor Stinner 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/4fac7ed43ebf1771a8fe86fdfe7b9991f3… commit: 4fac7ed43ebf1771a8fe86fdfe7b9991f3be78cd branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-12T13:02:29+01:00 summary: bpo-21016: pydoc and trace use sysconfig (GH-18476) bpo-21016, bpo-1294959: The pydoc and trace modules now use the sysconfig module to get the path to the Python standard library, to support uncommon installation path like /usr/lib64/python3.9/ on Fedora. Co-Authored-By: Jan Matějek <jmatejek(a)suse.com> files: A Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst M Lib/pydoc.py M Lib/trace.py diff --git a/Lib/pydoc.py b/Lib/pydoc.py index e32fdf76978e2..f172700a15f9d 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -66,6 +66,7 @@ class or function within a module or module in a package. If the import platform import re import sys +import sysconfig import time import tokenize import urllib.parse @@ -392,9 +393,7 @@ def fail(self, object, name=None, *args): docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, - basedir=os.path.join(sys.base_exec_prefix, "lib", - "python%d.%d" % sys.version_info[:2])): + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: diff --git a/Lib/trace.py b/Lib/trace.py index 681c3f9d05f81..52047c3fbf473 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -52,6 +52,7 @@ import linecache import os import sys +import sysconfig import token import tokenize import inspect @@ -660,9 +661,8 @@ def main(): opts = parser.parse_args() if opts.ignore_dir: - rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info) - _prefix = os.path.join(sys.base_prefix, *rel_path) - _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path) + _prefix = sysconfig.get_path("stdlib") + _exec_prefix = sysconfig.get_path("platstdlib") def parse_ignore_dir(s): s = os.path.expanduser(os.path.expandvars(s)) diff --git a/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst b/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst new file mode 100644 index 0000000000000..fb91bb3825555 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst @@ -0,0 +1,4 @@ +The :mod:`pydoc` and :mod:`trace` modules now use the :mod:`sysconfig` +module to get the path to the Python standard library, to support uncommon +installation path like ``/usr/lib64/python3.9/`` on Fedora. +Patch by Jan Matějek.

1 0

bpo-39219: Fix SyntaxError attributes in the tokenizer. (GH-17828)
by Miss Islington (bot) 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/efd878cdb46d9c7038d93fb36eb1ff7dc5… commit: efd878cdb46d9c7038d93fb36eb1ff7dc5baf9ec branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T02:35:10-08:00 summary: bpo-39219: Fix SyntaxError attributes in the tokenizer. (GH-17828) * Always set the text attribute. * Correct the offset attribute for non-ascii sources. (cherry picked from commit 0cc6b5e559b8303b18fdd56c2befd900fe7b5e35) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst M Lib/test/test_exceptions.py M Parser/tokenizer.c diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 10c1e076464e2..3a32253157369 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -179,17 +179,25 @@ def ckmsg(src, msg, exception=SyntaxError): ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError) def testSyntaxErrorOffset(self): - def check(src, lineno, offset): + def check(src, lineno, offset, encoding='utf-8'): with self.assertRaises(SyntaxError) as cm: compile(src, '<fragment>', 'exec') self.assertEqual(cm.exception.lineno, lineno) self.assertEqual(cm.exception.offset, offset) + if cm.exception.text is not None: + if not isinstance(src, str): + src = src.decode(encoding, 'replace') + line = src.split('\n')[lineno-1] + self.assertEqual(cm.exception.text.rstrip('\n'), line) check('def fact(x):\n\treturn x!\n', 2, 10) check('1 +\n', 1, 4) check('def spam():\n print(1)\n print(2)', 3, 10) check('Python = "Python" +', 1, 20) check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) + check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +', + 2, 19, encoding='cp1251') + check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 18) check('x = "a', 1, 7) check('lambda x: x = 2', 1, 1) @@ -205,6 +213,10 @@ def check(src, lineno, offset): check('0010 + 2', 1, 4) check('x = 32e-+4', 1, 8) check('x = 0o9', 1, 6) + check('\u03b1 = 0xI', 1, 6) + check(b'\xce\xb1 = 0xI', 1, 6) + check(b'# -*- coding: iso8859-7 -*-\n\xe1 = 0xI', 2, 6, + encoding='iso8859-7') # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 5) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst new file mode 100644 index 0000000000000..dac8360df712c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst @@ -0,0 +1,2 @@ +Syntax errors raised in the tokenizer now always set correct "text" and +"offset" attributes. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f73c32684c7b7..aecbcebb917e8 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1,6 +1,7 @@ /* Tokenizer implementation */ +#define PY_SSIZE_T_CLEAN #include "Python.h" #include <ctype.h> @@ -1034,17 +1035,44 @@ tok_backup(struct tok_state *tok, int c) static int syntaxerror(struct tok_state *tok, const char *format, ...) { + PyObject *errmsg, *errtext, *args; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES va_start(vargs, format); #else va_start(vargs); #endif - PyErr_FormatV(PyExc_SyntaxError, format, vargs); + errmsg = PyUnicode_FromFormatV(format, vargs); va_end(vargs); - PyErr_SyntaxLocationObject(tok->filename, - tok->lineno, - (int)(tok->cur - tok->line_start)); + if (!errmsg) { + goto error; + } + + errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start, + "replace"); + if (!errtext) { + goto error; + } + int offset = (int)PyUnicode_GET_LENGTH(errtext); + Py_ssize_t line_len = strcspn(tok->line_start, "\n"); + if (line_len != tok->cur - tok->line_start) { + Py_DECREF(errtext); + errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len, + "replace"); + } + if (!errtext) { + goto error; + } + + args = Py_BuildValue("(O(OiiN))", errmsg, + tok->filename, tok->lineno, offset, errtext); + if (args) { + PyErr_SetObject(PyExc_SyntaxError, args); + Py_DECREF(args); + } + +error: + Py_XDECREF(errmsg); tok->done = E_ERROR; return ERRORTOKEN; }

1 0

bpo-32856: Optimize the assignment idiom in comprehensions. (GH-16814)
by Serhiy Storchaka 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/8c579b1cc86053473eb052b76327279476… commit: 8c579b1cc86053473eb052b76327279476740c9b branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T12:18:59+02:00 summary: bpo-32856: Optimize the assignment idiom in comprehensions. (GH-16814) Now `for y in [expr]` in comprehensions is as fast as a simple assignment `y = expr`. files: A Misc/NEWS.d/next/Core and Builtins/2018-02-16-10-44-24.bpo-32856.UjR8SD.rst M Doc/whatsnew/3.9.rst M Lib/test/test_dictcomps.py M Lib/test/test_genexps.py M Lib/test/test_listcomps.py M Lib/test/test_peepholer.py M Lib/test/test_setcomps.py M Python/compile.c diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index c0404101d7ce2..ec179845aee7c 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -315,6 +315,17 @@ case), and one used ``__VENV_NAME__`` instead. Optimizations ============= +* Optimized the idiom for assignment a temporary variable in comprehensions. + Now ``for y in [expr]`` in comprehensions is as fast as a simple assignment + ``y = expr``. For example: + + sums = [s for s in [0] for x in data for s in [s + x]] + + Unlike to the ``:=`` operator this idiom does not leak a variable to the + outer scope. + + (Contributed by Serhiy Storchaka in :issue:`32856`.) + Build and C API Changes ======================= diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 927e3103e664b..16aa651b93c46 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -111,5 +111,22 @@ def add_call(pos, value): self.assertEqual(actual, expected) self.assertEqual(actual_calls, expected_calls) + def test_assignment_idiom_in_comprehensions(self): + expected = {1: 1, 2: 4, 3: 9, 4: 16} + actual = {j: j*j for i in range(4) for j in [i+1]} + self.assertEqual(actual, expected) + expected = {3: 2, 5: 6, 7: 12, 9: 20} + actual = {j+k: j*k for i in range(4) for j in [i+1] for k in [j+1]} + self.assertEqual(actual, expected) + expected = {3: 2, 5: 6, 7: 12, 9: 20} + actual = {j+k: j*k for i in range(4) for j, k in [(i+1, i+2)]} + self.assertEqual(actual, expected) + + def test_star_expression(self): + expected = {0: 0, 1: 1, 2: 4, 3: 9} + self.assertEqual({i: i*i for i in [*range(4)]}, expected) + self.assertEqual({i: i*i for i in (*range(4),)}, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index fd712bb172d5d..86e4e195f55ec 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -15,6 +15,22 @@ >>> list((i,j) for i in range(4) for j in range(i) ) [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] +Test the idiom for temporary variable assignment in comprehensions. + + >>> list((j*j for i in range(4) for j in [i+1])) + [1, 4, 9, 16] + >>> list((j*k for i in range(4) for j in [i+1] for k in [j+1])) + [2, 6, 12, 20] + >>> list((j*k for i in range(4) for j, k in [(i+1, i+2)])) + [2, 6, 12, 20] + +Not assignment + + >>> list((i*i for i in [*range(4)])) + [0, 1, 4, 9] + >>> list((i*i for i in (*range(4),))) + [0, 1, 4, 9] + Make sure the induction variable is not exposed >>> i = 20 diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index ddb169fe58957..62b3319ad936d 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -16,6 +16,22 @@ >>> [(i,j) for i in range(4) for j in range(i)] [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] +Test the idiom for temporary variable assignment in comprehensions. + + >>> [j*j for i in range(4) for j in [i+1]] + [1, 4, 9, 16] + >>> [j*k for i in range(4) for j in [i+1] for k in [j+1]] + [2, 6, 12, 20] + >>> [j*k for i in range(4) for j, k in [(i+1, i+2)]] + [2, 6, 12, 20] + +Not assignment + + >>> [i*i for i in [*range(4)]] + [0, 1, 4, 9] + >>> [i*i for i in (*range(4),)] + [0, 1, 4, 9] + Make sure the induction variable is not exposed >>> i = 20 diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 567e6a14361d1..7913e91e453ca 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -495,6 +495,20 @@ def f(x): return 6 self.check_lnotab(f) + def test_assignment_idiom_in_comprehensions(self): + def listcomp(): + return [y for x in a for y in [f(x)]] + self.assertEqual(count_instr_recursively(listcomp, 'FOR_ITER'), 1) + def setcomp(): + return {y for x in a for y in [f(x)]} + self.assertEqual(count_instr_recursively(setcomp, 'FOR_ITER'), 1) + def dictcomp(): + return {y: y for x in a for y in [f(x)]} + self.assertEqual(count_instr_recursively(dictcomp, 'FOR_ITER'), 1) + def genexpr(): + return (y for x in a for y in [f(x)]) + self.assertEqual(count_instr_recursively(genexpr, 'FOR_ITER'), 1) + class TestBuglets(unittest.TestCase): diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index fb7cde03d7823..ecc4fffec0d84 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -21,6 +21,22 @@ >>> list(sorted({(i,j) for i in range(4) for j in range(i)})) [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] +Test the idiom for temporary variable assignment in comprehensions. + + >>> sorted({j*j for i in range(4) for j in [i+1]}) + [1, 4, 9, 16] + >>> sorted({j*k for i in range(4) for j in [i+1] for k in [j+1]}) + [2, 6, 12, 20] + >>> sorted({j*k for i in range(4) for j, k in [(i+1, i+2)]}) + [2, 6, 12, 20] + +Not assignment + + >>> sorted({i*i for i in [*range(4)]}) + [0, 1, 4, 9] + >>> sorted({i*i for i in (*range(4),)}) + [0, 1, 4, 9] + Make sure the induction variable is not exposed >>> i = 20 diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-16-10-44-24.bpo-32856.UjR8SD.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-16-10-44-24.bpo-32856.UjR8SD.rst new file mode 100644 index 0000000000000..c1cd68f672712 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-16-10-44-24.bpo-32856.UjR8SD.rst @@ -0,0 +1,3 @@ +Optimized the idiom for assignment a temporary variable in comprehensions. +Now ``for y in [expr]`` in comprehensions is as fast as a simple assignment +``y = expr``. diff --git a/Python/compile.c b/Python/compile.c index 04b8fe46e194d..bf8c8109d0758 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -212,11 +212,13 @@ static int compiler_set_qualname(struct compiler *); static int compiler_sync_comprehension_generator( struct compiler *c, asdl_seq *generators, int gen_index, + int depth, expr_ty elt, expr_ty val, int type); static int compiler_async_comprehension_generator( struct compiler *c, asdl_seq *generators, int gen_index, + int depth, expr_ty elt, expr_ty val, int type); static PyCodeObject *assemble(struct compiler *, int addNone); @@ -4343,22 +4345,24 @@ compiler_call_helper(struct compiler *c, static int compiler_comprehension_generator(struct compiler *c, asdl_seq *generators, int gen_index, + int depth, expr_ty elt, expr_ty val, int type) { comprehension_ty gen; gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); if (gen->is_async) { return compiler_async_comprehension_generator( - c, generators, gen_index, elt, val, type); + c, generators, gen_index, depth, elt, val, type); } else { return compiler_sync_comprehension_generator( - c, generators, gen_index, elt, val, type); + c, generators, gen_index, depth, elt, val, type); } } static int compiler_sync_comprehension_generator(struct compiler *c, asdl_seq *generators, int gen_index, + int depth, expr_ty elt, expr_ty val, int type) { /* generate code for the iterator, then each of the ifs, @@ -4386,12 +4390,38 @@ compiler_sync_comprehension_generator(struct compiler *c, } else { /* Sub-iter - calculate on the fly */ - VISIT(c, expr, gen->iter); - ADDOP(c, GET_ITER); + /* Fast path for the temporary variable assignment idiom: + for y in [f(x)] + */ + asdl_seq *elts; + switch (gen->iter->kind) { + case List_kind: + elts = gen->iter->v.List.elts; + break; + case Tuple_kind: + elts = gen->iter->v.Tuple.elts; + break; + default: + elts = NULL; + } + if (asdl_seq_LEN(elts) == 1) { + expr_ty elt = asdl_seq_GET(elts, 0); + if (elt->kind != Starred_kind) { + VISIT(c, expr, elt); + start = NULL; + } + } + if (start) { + VISIT(c, expr, gen->iter); + ADDOP(c, GET_ITER); + } + } + if (start) { + depth++; + compiler_use_next_block(c, start); + ADDOP_JREL(c, FOR_ITER, anchor); + NEXT_BLOCK(c); } - compiler_use_next_block(c, start); - ADDOP_JREL(c, FOR_ITER, anchor); - NEXT_BLOCK(c); VISIT(c, expr, gen->target); /* XXX this needs to be cleaned up...a lot! */ @@ -4405,7 +4435,7 @@ compiler_sync_comprehension_generator(struct compiler *c, if (++gen_index < asdl_seq_LEN(generators)) if (!compiler_comprehension_generator(c, - generators, gen_index, + generators, gen_index, depth, elt, val, type)) return 0; @@ -4420,18 +4450,18 @@ compiler_sync_comprehension_generator(struct compiler *c, break; case COMP_LISTCOMP: VISIT(c, expr, elt); - ADDOP_I(c, LIST_APPEND, gen_index + 1); + ADDOP_I(c, LIST_APPEND, depth + 1); break; case COMP_SETCOMP: VISIT(c, expr, elt); - ADDOP_I(c, SET_ADD, gen_index + 1); + ADDOP_I(c, SET_ADD, depth + 1); break; case COMP_DICTCOMP: /* With '{k: v}', k is evaluated before v, so we do the same. */ VISIT(c, expr, elt); VISIT(c, expr, val); - ADDOP_I(c, MAP_ADD, gen_index + 1); + ADDOP_I(c, MAP_ADD, depth + 1); break; default: return 0; @@ -4440,8 +4470,10 @@ compiler_sync_comprehension_generator(struct compiler *c, compiler_use_next_block(c, skip); } compiler_use_next_block(c, if_cleanup); - ADDOP_JABS(c, JUMP_ABSOLUTE, start); - compiler_use_next_block(c, anchor); + if (start) { + ADDOP_JABS(c, JUMP_ABSOLUTE, start); + compiler_use_next_block(c, anchor); + } return 1; } @@ -4449,6 +4481,7 @@ compiler_sync_comprehension_generator(struct compiler *c, static int compiler_async_comprehension_generator(struct compiler *c, asdl_seq *generators, int gen_index, + int depth, expr_ty elt, expr_ty val, int type) { comprehension_ty gen; @@ -4492,9 +4525,10 @@ compiler_async_comprehension_generator(struct compiler *c, NEXT_BLOCK(c); } + depth++; if (++gen_index < asdl_seq_LEN(generators)) if (!compiler_comprehension_generator(c, - generators, gen_index, + generators, gen_index, depth, elt, val, type)) return 0; @@ -4509,18 +4543,18 @@ compiler_async_comprehension_generator(struct compiler *c, break; case COMP_LISTCOMP: VISIT(c, expr, elt); - ADDOP_I(c, LIST_APPEND, gen_index + 1); + ADDOP_I(c, LIST_APPEND, depth + 1); break; case COMP_SETCOMP: VISIT(c, expr, elt); - ADDOP_I(c, SET_ADD, gen_index + 1); + ADDOP_I(c, SET_ADD, depth + 1); break; case COMP_DICTCOMP: /* With '{k: v}', k is evaluated before v, so we do the same. */ VISIT(c, expr, elt); VISIT(c, expr, val); - ADDOP_I(c, MAP_ADD, gen_index + 1); + ADDOP_I(c, MAP_ADD, depth + 1); break; default: return 0; @@ -4583,7 +4617,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP_I(c, op, 0); } - if (!compiler_comprehension_generator(c, generators, 0, elt, + if (!compiler_comprehension_generator(c, generators, 0, 0, elt, val, type)) goto error_in_scope;

1 0

bpo-39219: Fix SyntaxError attributes in the tokenizer. (GH-17828)
by Serhiy Storchaka 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/0cc6b5e559b8303b18fdd56c2befd900fe… commit: 0cc6b5e559b8303b18fdd56c2befd900fe7b5e35 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T12:17:00+02:00 summary: bpo-39219: Fix SyntaxError attributes in the tokenizer. (GH-17828) * Always set the text attribute. * Correct the offset attribute for non-ascii sources. files: A Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst M Lib/test/test_exceptions.py M Parser/tokenizer.c diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 4d1aa4bca623f..22a22363a7d43 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -179,17 +179,25 @@ def ckmsg(src, msg, exception=SyntaxError): ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError) def testSyntaxErrorOffset(self): - def check(src, lineno, offset): + def check(src, lineno, offset, encoding='utf-8'): with self.assertRaises(SyntaxError) as cm: compile(src, '<fragment>', 'exec') self.assertEqual(cm.exception.lineno, lineno) self.assertEqual(cm.exception.offset, offset) + if cm.exception.text is not None: + if not isinstance(src, str): + src = src.decode(encoding, 'replace') + line = src.split('\n')[lineno-1] + self.assertEqual(cm.exception.text.rstrip('\n'), line) check('def fact(x):\n\treturn x!\n', 2, 10) check('1 +\n', 1, 4) check('def spam():\n print(1)\n print(2)', 3, 10) check('Python = "Python" +', 1, 20) check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) + check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +', + 2, 19, encoding='cp1251') + check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 18) check('x = "a', 1, 7) check('lambda x: x = 2', 1, 1) @@ -205,6 +213,10 @@ def check(src, lineno, offset): check('0010 + 2', 1, 4) check('x = 32e-+4', 1, 8) check('x = 0o9', 1, 6) + check('\u03b1 = 0xI', 1, 6) + check(b'\xce\xb1 = 0xI', 1, 6) + check(b'# -*- coding: iso8859-7 -*-\n\xe1 = 0xI', 2, 6, + encoding='iso8859-7') # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 5) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst new file mode 100644 index 0000000000000..dac8360df712c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-05-13-36-08.bpo-39219.uHtKd4.rst @@ -0,0 +1,2 @@ +Syntax errors raised in the tokenizer now always set correct "text" and +"offset" attributes. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index c37cd927df5a4..630b0aaab03f9 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1,6 +1,7 @@ /* Tokenizer implementation */ +#define PY_SSIZE_T_CLEAN #include "Python.h" #include <ctype.h> @@ -1034,17 +1035,44 @@ tok_backup(struct tok_state *tok, int c) static int syntaxerror(struct tok_state *tok, const char *format, ...) { + PyObject *errmsg, *errtext, *args; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES va_start(vargs, format); #else va_start(vargs); #endif - PyErr_FormatV(PyExc_SyntaxError, format, vargs); + errmsg = PyUnicode_FromFormatV(format, vargs); va_end(vargs); - PyErr_SyntaxLocationObject(tok->filename, - tok->lineno, - (int)(tok->cur - tok->line_start)); + if (!errmsg) { + goto error; + } + + errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start, + "replace"); + if (!errtext) { + goto error; + } + int offset = (int)PyUnicode_GET_LENGTH(errtext); + Py_ssize_t line_len = strcspn(tok->line_start, "\n"); + if (line_len != tok->cur - tok->line_start) { + Py_DECREF(errtext); + errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len, + "replace"); + } + if (!errtext) { + goto error; + } + + args = Py_BuildValue("(O(OiiN))", errmsg, + tok->filename, tok->lineno, offset, errtext); + if (args) { + PyErr_SetObject(PyExc_SyntaxError, args); + Py_DECREF(args); + } + +error: + Py_XDECREF(errmsg); tok->done = E_ERROR; return ERRORTOKEN; }

1 0

bpo-39567: Add audit for os.walk(), os.fwalk(), Path.glob() and Path.rglob(). (GH-18372)
by Serhiy Storchaka 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/f4f445b69306c68a2ba8ce8eb8c6cb3064… commit: f4f445b69306c68a2ba8ce8eb8c6cb3064db5fe7 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T12:11:34+02:00 summary: bpo-39567: Add audit for os.walk(), os.fwalk(), Path.glob() and Path.rglob(). (GH-18372) files: A Misc/NEWS.d/next/Library/2020-02-06-10-23-32.bpo-39567.VpFBxt.rst M Lib/os.py M Lib/pathlib.py diff --git a/Lib/os.py b/Lib/os.py index 7ae102617e8b1..ab75b94d4fe45 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -336,7 +336,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirs.remove('CVS') # don't visit CVS directories """ - top = fspath(top) + sys.audit("os.walk", top, topdown, onerror, followlinks) + return _walk(fspath(top), topdown, onerror, followlinks) + +def _walk(top, topdown, onerror, followlinks): dirs = [] nondirs = [] walk_dirs = [] @@ -410,11 +413,11 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # the caller can replace the directory entry during the "yield" # above. if followlinks or not islink(new_path): - yield from walk(new_path, topdown, onerror, followlinks) + yield from _walk(new_path, topdown, onerror, followlinks) else: # Recurse into sub-directories for new_path in walk_dirs: - yield from walk(new_path, topdown, onerror, followlinks) + yield from _walk(new_path, topdown, onerror, followlinks) # Yield after recursion if going bottom up yield top, dirs, nondirs @@ -455,6 +458,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories """ + sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) if not isinstance(top, int) or not hasattr(top, '__index__'): top = fspath(top) # Note: To guard against symlink races, we use the standard diff --git a/Lib/pathlib.py b/Lib/pathlib.py index a5f3313902e1b..cfa574af6e8ba 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1134,6 +1134,7 @@ def glob(self, pattern): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ + sys.audit("pathlib.Path.glob", self, pattern) if not pattern: raise ValueError("Unacceptable pattern: {!r}".format(pattern)) drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) @@ -1148,6 +1149,7 @@ def rglob(self, pattern): directories) matching the given relative pattern, anywhere in this subtree. """ + sys.audit("pathlib.Path.rglob", self, pattern) drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") diff --git a/Misc/NEWS.d/next/Library/2020-02-06-10-23-32.bpo-39567.VpFBxt.rst b/Misc/NEWS.d/next/Library/2020-02-06-10-23-32.bpo-39567.VpFBxt.rst new file mode 100644 index 0000000000000..3c4700f455b5e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-06-10-23-32.bpo-39567.VpFBxt.rst @@ -0,0 +1,2 @@ +Added audit for :func:`os.walk`, :func:`os.fwalk`, :meth:`pathlib.Path.glob` +and :meth:`pathlib.Path.rglob`.

1 0

bpo-39605: Remove a cast that causes a warning. (GH-18473)
by Miss Islington (bot) 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/0b8f738eb3ee0110461e7da28c0b6b452f… commit: 0b8f738eb3ee0110461e7da28c0b6b452f91999d branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-11T19:52:46-08:00 summary: bpo-39605: Remove a cast that causes a warning. (GH-18473) (cherry picked from commit 95905ce0f41fd42eb1ef60ddb83f057401c3d52f) Co-authored-by: Benjamin Peterson <benjamin(a)python.org> files: M Objects/unicodeobject.c diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8ba379e8887e9..4c2b42f959b83 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3353,7 +3353,7 @@ PyUnicode_Decode(const char *s, /* Decode via the codec registry */ buffer = NULL; - if (PyBuffer_FillInfo(&info, NULL, (const void *)s, size, 1, PyBUF_FULL_RO) < 0) + if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0) goto onError; buffer = PyMemoryView_FromBuffer(&info); if (buffer == NULL)

1 0

bpo-39605: Remove a cast that causes a warning. (GH-18473)
by Benjamin Peterson 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/95905ce0f41fd42eb1ef60ddb83f057401… commit: 95905ce0f41fd42eb1ef60ddb83f057401c3d52f branch: master author: Benjamin Peterson <benjamin(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-11T19:36:14-08:00 summary: bpo-39605: Remove a cast that causes a warning. (GH-18473) files: M Objects/unicodeobject.c diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index fdc2ca6612cc5..8470e41624bd4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3420,7 +3420,7 @@ PyUnicode_Decode(const char *s, /* Decode via the codec registry */ buffer = NULL; - if (PyBuffer_FillInfo(&info, NULL, (const void *)s, size, 1, PyBUF_FULL_RO) < 0) + if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0) goto onError; buffer = PyMemoryView_FromBuffer(&info); if (buffer == NULL)

1 0

bpo-39595: Improve zipfile.Path performance (GH-18406) (GH-18472)
by Miss Islington (bot) 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/ed4d263e8767b0e4c47df99141b500d36c… commit: ed4d263e8767b0e4c47df99141b500d36ce0275d branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-11T22:21:32-05:00 summary: bpo-39595: Improve zipfile.Path performance (GH-18406) (GH-18472) * Improve zipfile.Path performance on zipfiles with a large number of entries. * 📜🤖 Added by blurb_it. * Add bpo to blurb * Sync with importlib_metadata 1.5 (6fe70ca) * Update blurb. * Remove compatibility code * Add stubs module, omitted from earlier commit Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot](a)users.noreply.github.com> (cherry picked from commit e5bd73632e77dc5ab0cab77e48e94ca5e354be8a) Co-authored-by: Jason R. Coombs <jaraco(a)jaraco.com> Co-authored-by: Jason R. Coombs <jaraco(a)jaraco.com> files: A Lib/test/test_importlib/stubs.py A Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst M Lib/importlib/metadata.py M Lib/test/test_importlib/fixtures.py M Lib/test/test_importlib/test_main.py M Lib/test/test_zipfile.py M Lib/zipfile.py diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py index ae8ecf9b8500c..831f593277ccd 100644 --- a/Lib/importlib/metadata.py +++ b/Lib/importlib/metadata.py @@ -391,6 +391,7 @@ class FastPath: def __init__(self, root): self.root = root + self.base = os.path.basename(root).lower() def joinpath(self, child): return pathlib.Path(self.root, child) @@ -413,12 +414,11 @@ def zip_children(self): ) def is_egg(self, search): - root_n_low = os.path.split(self.root)[1].lower() - + base = self.base return ( - root_n_low == search.normalized + '.egg' - or root_n_low.startswith(search.prefix) - and root_n_low.endswith('.egg')) + base == search.versionless_egg_name + or base.startswith(search.prefix) + and base.endswith('.egg')) def search(self, name): for child in self.children(): @@ -439,6 +439,7 @@ class Prepared: prefix = '' suffixes = '.dist-info', '.egg-info' exact_matches = [''][:0] + versionless_egg_name = '' def __init__(self, name): self.name = name @@ -448,6 +449,7 @@ def __init__(self, name): self.prefix = self.normalized + '-' self.exact_matches = [ self.normalized + suffix for suffix in self.suffixes] + self.versionless_egg_name = self.normalized + '.egg' class MetadataPathFinder(DistributionFinder): diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index 0b4ce18d5a6cd..695c92a786cb0 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -47,14 +47,28 @@ def tempdir_as_cwd(): yield tmp -class SiteDir: +(a)contextlib.contextmanager +def install_finder(finder): + sys.meta_path.append(finder) + try: + yield + finally: + sys.meta_path.remove(finder) + + +class Fixtures: def setUp(self): self.fixtures = ExitStack() self.addCleanup(self.fixtures.close) + + +class SiteDir(Fixtures): + def setUp(self): + super(SiteDir, self).setUp() self.site_dir = self.fixtures.enter_context(tempdir()) -class OnSysPath: +class OnSysPath(Fixtures): @staticmethod @contextlib.contextmanager def add_sys_path(dir): @@ -198,3 +212,8 @@ def build_files(file_defs, prefix=pathlib.Path()): def DALS(str): "Dedent and left-strip" return textwrap.dedent(str).lstrip() + + +class NullFinder: + def find_module(self, name): + pass diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/stubs.py new file mode 100644 index 0000000000000..e5b011c399fa9 --- /dev/null +++ b/Lib/test/test_importlib/stubs.py @@ -0,0 +1,10 @@ +import unittest + + +class fake_filesystem_unittest: + """ + Stubbed version of the pyfakefs module + """ + class TestCase(unittest.TestCase): + def setUpPyfakefs(self): + self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index c5f1dbbae325e..42a79992ecc8c 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -7,6 +7,11 @@ import unittest import importlib.metadata +try: + import pyfakefs.fake_filesystem_unittest as ffs +except ImportError: + from .stubs import fake_filesystem_unittest as ffs + from . import fixtures from importlib.metadata import ( Distribution, EntryPoint, @@ -185,6 +190,33 @@ def test_egg(self): version('foo') +class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): + site_dir = '/does-not-exist' + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + importlib.metadata.distributions() + + +class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): + site_dir = '/access-denied' + + def setUp(self): + super(InaccessibleSysPath, self).setUp() + self.setUpPyfakefs() + self.fs.create_dir(self.site_dir, perm_bits=000) + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + list(importlib.metadata.distributions()) + + class TestEntryPoints(unittest.TestCase): def __init__(self, *args): super(TestEntryPoints, self).__init__(*args) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index c65de9202c0c4..61bca8651c02a 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2683,16 +2683,71 @@ def test_extract_command(self): self.assertEqual(f.read(), zf.read(zi)) +class TestExecutablePrependedZip(unittest.TestCase): + """Test our ability to open zip files with an executable prepended.""" + + def setUp(self): + self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata') + self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata') + + def _test_zip_works(self, name): + # bpo28494 sanity check: ensure is_zipfile works on these. + self.assertTrue(zipfile.is_zipfile(name), + f'is_zipfile failed on {name}') + # Ensure we can operate on these via ZipFile. + with zipfile.ZipFile(name) as zipfp: + for n in zipfp.namelist(): + data = zipfp.read(n) + self.assertIn(b'FAVORITE_NUMBER', data) + + def test_read_zip_with_exe_prepended(self): + self._test_zip_works(self.exe_zip) + + def test_read_zip64_with_exe_prepended(self): + self._test_zip_works(self.exe_zip64) + + @unittest.skipUnless(sys.executable, 'sys.executable required.') + @unittest.skipUnless(os.access('/bin/bash', os.X_OK), + 'Test relies on #!/bin/bash working.') + def test_execute_zip2(self): + output = subprocess.check_output([self.exe_zip, sys.executable]) + self.assertIn(b'number in executable: 5', output) + + @unittest.skipUnless(sys.executable, 'sys.executable required.') + @unittest.skipUnless(os.access('/bin/bash', os.X_OK), + 'Test relies on #!/bin/bash working.') + def test_execute_zip64(self): + output = subprocess.check_output([self.exe_zip64, sys.executable]) + self.assertIn(b'number in executable: 5', output) + + # Poor man's technique to consume a (smallish) iterable. consume = tuple +# from jaraco.itertools 5.0 +class jaraco: + class itertools: + class Counter: + def __init__(self, i): + self.count = 0 + self._orig_iter = iter(i) + + def __iter__(self): + return self + + def __next__(self): + result = next(self._orig_iter) + self.count += 1 + return result + + def add_dirs(zf): """ Given a writable zip file zf, inject directory entries for any directories implied by the presence of children. """ - for name in zipfile.Path._implied_dirs(zf.namelist()): + for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): zf.writestr(name, b"") return zf @@ -2733,44 +2788,6 @@ def build_alpharep_fixture(): return zf -class TestExecutablePrependedZip(unittest.TestCase): - """Test our ability to open zip files with an executable prepended.""" - - def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata') - - def _test_zip_works(self, name): - # bpo-28494 sanity check: ensure is_zipfile works on these. - self.assertTrue(zipfile.is_zipfile(name), - f'is_zipfile failed on {name}') - # Ensure we can operate on these via ZipFile. - with zipfile.ZipFile(name) as zipfp: - for n in zipfp.namelist(): - data = zipfp.read(n) - self.assertIn(b'FAVORITE_NUMBER', data) - - def test_read_zip_with_exe_prepended(self): - self._test_zip_works(self.exe_zip) - - def test_read_zip64_with_exe_prepended(self): - self._test_zip_works(self.exe_zip64) - - @unittest.skipUnless(sys.executable, 'sys.executable required.') - @unittest.skipUnless(os.access('/bin/bash', os.X_OK), - 'Test relies on #!/bin/bash working.') - def test_execute_zip2(self): - output = subprocess.check_output([self.exe_zip, sys.executable]) - self.assertIn(b'number in executable: 5', output) - - @unittest.skipUnless(sys.executable, 'sys.executable required.') - @unittest.skipUnless(os.access('/bin/bash', os.X_OK), - 'Test relies on #!/bin/bash working.') - def test_execute_zip64(self): - output = subprocess.check_output([self.exe_zip64, sys.executable]) - self.assertIn(b'number in executable: 5', output) - - class TestPath(unittest.TestCase): def setUp(self): self.fixtures = contextlib.ExitStack() @@ -2808,6 +2825,14 @@ def test_iterdir_and_types(self): i, = h.iterdir() assert i.is_file() + def test_subdir_is_dir(self): + for alpharep in self.zipfile_alpharep(): + root = zipfile.Path(alpharep) + assert (root / 'b').is_dir() + assert (root / 'b/').is_dir() + assert (root / 'g').is_dir() + assert (root / 'g/').is_dir() + def test_open(self): for alpharep in self.zipfile_alpharep(): root = zipfile.Path(alpharep) @@ -2869,6 +2894,45 @@ def test_missing_dir_parent(self): root = zipfile.Path(alpharep) assert (root / 'missing dir/').parent.at == '' + def test_mutability(self): + """ + If the underlying zipfile is changed, the Path object should + reflect that change. + """ + for alpharep in self.zipfile_alpharep(): + root = zipfile.Path(alpharep) + a, b, g = root.iterdir() + alpharep.writestr('foo.txt', 'foo') + alpharep.writestr('bar/baz.txt', 'baz') + assert any( + child.name == 'foo.txt' + for child in root.iterdir()) + assert (root / 'foo.txt').read_text() == 'foo' + baz, = (root / 'bar').iterdir() + assert baz.read_text() == 'baz' + + HUGE_ZIPFILE_NUM_ENTRIES = 2 ** 13 + + def huge_zipfile(self): + """Create a read-only zipfile with a huge number of entries entries.""" + strm = io.BytesIO() + zf = zipfile.ZipFile(strm, "w") + for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): + zf.writestr(entry, entry) + zf.mode = 'r' + return zf + + def test_joinpath_constant_time(self): + """ + Ensure joinpath on items in zipfile is linear time. + """ + root = zipfile.Path(self.huge_zipfile()) + entries = jaraco.itertools.Counter(root.iterdir()) + for entry in entries: + entry.joinpath('suffix') + # Check the file iterated all items + assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES + if __name__ == "__main__": unittest.main() diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 8b99c1189baa8..5dc6516cc47b7 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -16,6 +16,8 @@ import sys import threading import time +import contextlib +from collections import OrderedDict try: import zlib # We may need its compression method @@ -2182,6 +2184,79 @@ def _ancestry(path): path, tail = posixpath.split(path) +class CompleteDirs(ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. + """ + + @staticmethod + def _implied_dirs(names): + parents = itertools.chain.from_iterable(map(_parents, names)) + # Deduplicate entries in original order + implied_dirs = OrderedDict.fromkeys( + p + posixpath.sep for p in parents + # Cast names to a set for O(1) lookups + if p + posixpath.sep not in set(names) + ) + return implied_dirs + + def namelist(self): + names = super(CompleteDirs, self).namelist() + return names + list(self._implied_dirs(names)) + + def _name_set(self): + return set(self.namelist()) + + def resolve_dir(self, name): + """ + If the name represents a directory, return that name + as a directory (with the trailing slash). + """ + names = self._name_set() + dirname = name + '/' + dir_match = name not in names and dirname in names + return dirname if dir_match else name + + @classmethod + def make(cls, source): + """ + Given a source (filename or zipfile), return an + appropriate CompleteDirs subclass. + """ + if isinstance(source, CompleteDirs): + return source + + if not isinstance(source, ZipFile): + return cls(source) + + # Only allow for FastPath when supplied zipfile is read-only + if 'r' not in source.mode: + cls = CompleteDirs + + res = cls.__new__(cls) + vars(res).update(vars(source)) + return res + + +class FastLookup(CompleteDirs): + """ + ZipFile subclass to ensure implicit + dirs exist and are resolved rapidly. + """ + def namelist(self): + with contextlib.suppress(AttributeError): + return self.__names + self.__names = super(FastLookup, self).namelist() + return self.__names + + def _name_set(self): + with contextlib.suppress(AttributeError): + return self.__lookup + self.__lookup = super(FastLookup, self)._name_set() + return self.__lookup + + class Path: """ A pathlib-compatible interface for zip files. @@ -2250,7 +2325,7 @@ class Path: __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" def __init__(self, root, at=""): - self.root = root if isinstance(root, ZipFile) else ZipFile(root) + self.root = FastLookup.make(root) self.at = at @property @@ -2282,12 +2357,12 @@ def is_file(self): return not self.is_dir() def exists(self): - return self.at in self._names() + return self.at in self.root._name_set() def iterdir(self): if not self.is_dir(): raise ValueError("Can't listdir a file") - subs = map(self._next, self._names()) + subs = map(self._next, self.root.namelist()) return filter(self._is_child, subs) def __str__(self): @@ -2298,25 +2373,10 @@ def __repr__(self): def joinpath(self, add): next = posixpath.join(self.at, add) - next_dir = posixpath.join(self.at, add, "") - names = self._names() - return self._next(next_dir if next not in names and next_dir in names else next) + return self._next(self.root.resolve_dir(next)) __truediv__ = joinpath - @staticmethod - def _implied_dirs(names): - return _unique_everseen( - parent + "/" - for name in names - for parent in _parents(name) - if parent + "/" not in names - ) - - @classmethod - def _add_implied_dirs(cls, names): - return names + list(cls._implied_dirs(names)) - @property def parent(self): parent_at = posixpath.dirname(self.at.rstrip('/')) @@ -2324,9 +2384,6 @@ def parent(self): parent_at += '/' return self._next(parent_at) - def _names(self): - return self._add_implied_dirs(self.root.namelist()) - def main(args=None): import argparse @@ -2388,5 +2445,6 @@ def addToZip(zf, path, zippath): zippath = '' addToZip(zf, path, zippath) + if __name__ == "__main__": main() diff --git a/Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst b/Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst new file mode 100644 index 0000000000000..3a461389af7d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst @@ -0,0 +1 @@ +Improved performance of zipfile.Path for files with a large number of entries. Also improved performance and fixed minor issue as published with `importlib_metadata 1.5 <https://importlib-metadata.readthedocs.io/en/latest/changelog%20(links).htm…>`_.

1 0

bpo-39595: Improve zipfile.Path performance (#18406)
by Jason R. Coombs 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/e5bd73632e77dc5ab0cab77e48e94ca5e3… commit: e5bd73632e77dc5ab0cab77e48e94ca5e354be8a branch: master author: Jason R. Coombs <jaraco(a)jaraco.com> committer: GitHub <noreply(a)github.com> date: 2020-02-11T21:58:47-05:00 summary: bpo-39595: Improve zipfile.Path performance (#18406) * Improve zipfile.Path performance on zipfiles with a large number of entries. * 📜🤖 Added by blurb_it. * Add bpo to blurb * Sync with importlib_metadata 1.5 (6fe70ca) * Update blurb. * Remove compatibility code * Add stubs module, omitted from earlier commit Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot](a)users.noreply.github.com> files: A Lib/test/test_importlib/stubs.py A Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst M Lib/importlib/metadata.py M Lib/test/test_importlib/fixtures.py M Lib/test/test_importlib/test_main.py M Lib/test/test_zipfile.py M Lib/zipfile.py diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py index ae8ecf9b8500c..831f593277ccd 100644 --- a/Lib/importlib/metadata.py +++ b/Lib/importlib/metadata.py @@ -391,6 +391,7 @@ class FastPath: def __init__(self, root): self.root = root + self.base = os.path.basename(root).lower() def joinpath(self, child): return pathlib.Path(self.root, child) @@ -413,12 +414,11 @@ def zip_children(self): ) def is_egg(self, search): - root_n_low = os.path.split(self.root)[1].lower() - + base = self.base return ( - root_n_low == search.normalized + '.egg' - or root_n_low.startswith(search.prefix) - and root_n_low.endswith('.egg')) + base == search.versionless_egg_name + or base.startswith(search.prefix) + and base.endswith('.egg')) def search(self, name): for child in self.children(): @@ -439,6 +439,7 @@ class Prepared: prefix = '' suffixes = '.dist-info', '.egg-info' exact_matches = [''][:0] + versionless_egg_name = '' def __init__(self, name): self.name = name @@ -448,6 +449,7 @@ def __init__(self, name): self.prefix = self.normalized + '-' self.exact_matches = [ self.normalized + suffix for suffix in self.suffixes] + self.versionless_egg_name = self.normalized + '.egg' class MetadataPathFinder(DistributionFinder): diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index 0b4ce18d5a6cd..695c92a786cb0 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -47,14 +47,28 @@ def tempdir_as_cwd(): yield tmp -class SiteDir: +(a)contextlib.contextmanager +def install_finder(finder): + sys.meta_path.append(finder) + try: + yield + finally: + sys.meta_path.remove(finder) + + +class Fixtures: def setUp(self): self.fixtures = ExitStack() self.addCleanup(self.fixtures.close) + + +class SiteDir(Fixtures): + def setUp(self): + super(SiteDir, self).setUp() self.site_dir = self.fixtures.enter_context(tempdir()) -class OnSysPath: +class OnSysPath(Fixtures): @staticmethod @contextlib.contextmanager def add_sys_path(dir): @@ -198,3 +212,8 @@ def build_files(file_defs, prefix=pathlib.Path()): def DALS(str): "Dedent and left-strip" return textwrap.dedent(str).lstrip() + + +class NullFinder: + def find_module(self, name): + pass diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/stubs.py new file mode 100644 index 0000000000000..e5b011c399fa9 --- /dev/null +++ b/Lib/test/test_importlib/stubs.py @@ -0,0 +1,10 @@ +import unittest + + +class fake_filesystem_unittest: + """ + Stubbed version of the pyfakefs module + """ + class TestCase(unittest.TestCase): + def setUpPyfakefs(self): + self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index c5f1dbbae325e..42a79992ecc8c 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -7,6 +7,11 @@ import unittest import importlib.metadata +try: + import pyfakefs.fake_filesystem_unittest as ffs +except ImportError: + from .stubs import fake_filesystem_unittest as ffs + from . import fixtures from importlib.metadata import ( Distribution, EntryPoint, @@ -185,6 +190,33 @@ def test_egg(self): version('foo') +class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): + site_dir = '/does-not-exist' + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + importlib.metadata.distributions() + + +class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): + site_dir = '/access-denied' + + def setUp(self): + super(InaccessibleSysPath, self).setUp() + self.setUpPyfakefs() + self.fs.create_dir(self.site_dir, perm_bits=000) + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + list(importlib.metadata.distributions()) + + class TestEntryPoints(unittest.TestCase): def __init__(self, *args): super(TestEntryPoints, self).__init__(*args) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index c334715f3d81b..09fc850600610 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2724,16 +2724,71 @@ def test_extract_command(self): self.assertEqual(f.read(), zf.read(zi)) +class TestExecutablePrependedZip(unittest.TestCase): + """Test our ability to open zip files with an executable prepended.""" + + def setUp(self): + self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata') + self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata') + + def _test_zip_works(self, name): + # bpo28494 sanity check: ensure is_zipfile works on these. + self.assertTrue(zipfile.is_zipfile(name), + f'is_zipfile failed on {name}') + # Ensure we can operate on these via ZipFile. + with zipfile.ZipFile(name) as zipfp: + for n in zipfp.namelist(): + data = zipfp.read(n) + self.assertIn(b'FAVORITE_NUMBER', data) + + def test_read_zip_with_exe_prepended(self): + self._test_zip_works(self.exe_zip) + + def test_read_zip64_with_exe_prepended(self): + self._test_zip_works(self.exe_zip64) + + @unittest.skipUnless(sys.executable, 'sys.executable required.') + @unittest.skipUnless(os.access('/bin/bash', os.X_OK), + 'Test relies on #!/bin/bash working.') + def test_execute_zip2(self): + output = subprocess.check_output([self.exe_zip, sys.executable]) + self.assertIn(b'number in executable: 5', output) + + @unittest.skipUnless(sys.executable, 'sys.executable required.') + @unittest.skipUnless(os.access('/bin/bash', os.X_OK), + 'Test relies on #!/bin/bash working.') + def test_execute_zip64(self): + output = subprocess.check_output([self.exe_zip64, sys.executable]) + self.assertIn(b'number in executable: 5', output) + + # Poor man's technique to consume a (smallish) iterable. consume = tuple +# from jaraco.itertools 5.0 +class jaraco: + class itertools: + class Counter: + def __init__(self, i): + self.count = 0 + self._orig_iter = iter(i) + + def __iter__(self): + return self + + def __next__(self): + result = next(self._orig_iter) + self.count += 1 + return result + + def add_dirs(zf): """ Given a writable zip file zf, inject directory entries for any directories implied by the presence of children. """ - for name in zipfile.Path._implied_dirs(zf.namelist()): + for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): zf.writestr(name, b"") return zf @@ -2774,44 +2829,6 @@ def build_alpharep_fixture(): return zf -class TestExecutablePrependedZip(unittest.TestCase): - """Test our ability to open zip files with an executable prepended.""" - - def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata') - - def _test_zip_works(self, name): - # bpo-28494 sanity check: ensure is_zipfile works on these. - self.assertTrue(zipfile.is_zipfile(name), - f'is_zipfile failed on {name}') - # Ensure we can operate on these via ZipFile. - with zipfile.ZipFile(name) as zipfp: - for n in zipfp.namelist(): - data = zipfp.read(n) - self.assertIn(b'FAVORITE_NUMBER', data) - - def test_read_zip_with_exe_prepended(self): - self._test_zip_works(self.exe_zip) - - def test_read_zip64_with_exe_prepended(self): - self._test_zip_works(self.exe_zip64) - - @unittest.skipUnless(sys.executable, 'sys.executable required.') - @unittest.skipUnless(os.access('/bin/bash', os.X_OK), - 'Test relies on #!/bin/bash working.') - def test_execute_zip2(self): - output = subprocess.check_output([self.exe_zip, sys.executable]) - self.assertIn(b'number in executable: 5', output) - - @unittest.skipUnless(sys.executable, 'sys.executable required.') - @unittest.skipUnless(os.access('/bin/bash', os.X_OK), - 'Test relies on #!/bin/bash working.') - def test_execute_zip64(self): - output = subprocess.check_output([self.exe_zip64, sys.executable]) - self.assertIn(b'number in executable: 5', output) - - class TestPath(unittest.TestCase): def setUp(self): self.fixtures = contextlib.ExitStack() @@ -2849,6 +2866,14 @@ def test_iterdir_and_types(self): i, = h.iterdir() assert i.is_file() + def test_subdir_is_dir(self): + for alpharep in self.zipfile_alpharep(): + root = zipfile.Path(alpharep) + assert (root / 'b').is_dir() + assert (root / 'b/').is_dir() + assert (root / 'g').is_dir() + assert (root / 'g/').is_dir() + def test_open(self): for alpharep in self.zipfile_alpharep(): root = zipfile.Path(alpharep) @@ -2910,6 +2935,45 @@ def test_missing_dir_parent(self): root = zipfile.Path(alpharep) assert (root / 'missing dir/').parent.at == '' + def test_mutability(self): + """ + If the underlying zipfile is changed, the Path object should + reflect that change. + """ + for alpharep in self.zipfile_alpharep(): + root = zipfile.Path(alpharep) + a, b, g = root.iterdir() + alpharep.writestr('foo.txt', 'foo') + alpharep.writestr('bar/baz.txt', 'baz') + assert any( + child.name == 'foo.txt' + for child in root.iterdir()) + assert (root / 'foo.txt').read_text() == 'foo' + baz, = (root / 'bar').iterdir() + assert baz.read_text() == 'baz' + + HUGE_ZIPFILE_NUM_ENTRIES = 2 ** 13 + + def huge_zipfile(self): + """Create a read-only zipfile with a huge number of entries entries.""" + strm = io.BytesIO() + zf = zipfile.ZipFile(strm, "w") + for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): + zf.writestr(entry, entry) + zf.mode = 'r' + return zf + + def test_joinpath_constant_time(self): + """ + Ensure joinpath on items in zipfile is linear time. + """ + root = zipfile.Path(self.huge_zipfile()) + entries = jaraco.itertools.Counter(root.iterdir()) + for entry in entries: + entry.joinpath('suffix') + # Check the file iterated all items + assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES + if __name__ == "__main__": unittest.main() diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 2da87ef505e6e..4510fac250b97 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -16,6 +16,8 @@ import sys import threading import time +import contextlib +from collections import OrderedDict try: import zlib # We may need its compression method @@ -2159,6 +2161,79 @@ def _ancestry(path): path, tail = posixpath.split(path) +class CompleteDirs(ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. + """ + + @staticmethod + def _implied_dirs(names): + parents = itertools.chain.from_iterable(map(_parents, names)) + # Deduplicate entries in original order + implied_dirs = OrderedDict.fromkeys( + p + posixpath.sep for p in parents + # Cast names to a set for O(1) lookups + if p + posixpath.sep not in set(names) + ) + return implied_dirs + + def namelist(self): + names = super(CompleteDirs, self).namelist() + return names + list(self._implied_dirs(names)) + + def _name_set(self): + return set(self.namelist()) + + def resolve_dir(self, name): + """ + If the name represents a directory, return that name + as a directory (with the trailing slash). + """ + names = self._name_set() + dirname = name + '/' + dir_match = name not in names and dirname in names + return dirname if dir_match else name + + @classmethod + def make(cls, source): + """ + Given a source (filename or zipfile), return an + appropriate CompleteDirs subclass. + """ + if isinstance(source, CompleteDirs): + return source + + if not isinstance(source, ZipFile): + return cls(source) + + # Only allow for FastPath when supplied zipfile is read-only + if 'r' not in source.mode: + cls = CompleteDirs + + res = cls.__new__(cls) + vars(res).update(vars(source)) + return res + + +class FastLookup(CompleteDirs): + """ + ZipFile subclass to ensure implicit + dirs exist and are resolved rapidly. + """ + def namelist(self): + with contextlib.suppress(AttributeError): + return self.__names + self.__names = super(FastLookup, self).namelist() + return self.__names + + def _name_set(self): + with contextlib.suppress(AttributeError): + return self.__lookup + self.__lookup = super(FastLookup, self)._name_set() + return self.__lookup + + class Path: """ A pathlib-compatible interface for zip files. @@ -2227,7 +2302,7 @@ class Path: __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" def __init__(self, root, at=""): - self.root = root if isinstance(root, ZipFile) else ZipFile(root) + self.root = FastLookup.make(root) self.at = at @property @@ -2259,12 +2334,12 @@ def is_file(self): return not self.is_dir() def exists(self): - return self.at in self._names() + return self.at in self.root._name_set() def iterdir(self): if not self.is_dir(): raise ValueError("Can't listdir a file") - subs = map(self._next, self._names()) + subs = map(self._next, self.root.namelist()) return filter(self._is_child, subs) def __str__(self): @@ -2275,25 +2350,10 @@ def __repr__(self): def joinpath(self, add): next = posixpath.join(self.at, add) - next_dir = posixpath.join(self.at, add, "") - names = self._names() - return self._next(next_dir if next not in names and next_dir in names else next) + return self._next(self.root.resolve_dir(next)) __truediv__ = joinpath - @staticmethod - def _implied_dirs(names): - return _unique_everseen( - parent + "/" - for name in names - for parent in _parents(name) - if parent + "/" not in names - ) - - @classmethod - def _add_implied_dirs(cls, names): - return names + list(cls._implied_dirs(names)) - @property def parent(self): parent_at = posixpath.dirname(self.at.rstrip('/')) @@ -2301,9 +2361,6 @@ def parent(self): parent_at += '/' return self._next(parent_at) - def _names(self): - return self._add_implied_dirs(self.root.namelist()) - def main(args=None): import argparse @@ -2365,5 +2422,6 @@ def addToZip(zf, path, zippath): zippath = '' addToZip(zf, path, zippath) + if __name__ == "__main__": main() diff --git a/Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst b/Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst new file mode 100644 index 0000000000000..3a461389af7d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-07-23-14-14.bpo-39595.DHwddE.rst @@ -0,0 +1 @@ +Improved performance of zipfile.Path for files with a large number of entries. Also improved performance and fixed minor issue as published with `importlib_metadata 1.5 <https://importlib-metadata.readthedocs.io/en/latest/changelog%20(links).htm…>`_.

1 0

closes bpo-39605: Fix some casts to not cast away const. (GH-18453)
by Miss Islington (bot) 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/190433d8150bf719fa0ba972dbacf22149… commit: 190433d8150bf719fa0ba972dbacf2214942f54e branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-11T18:47:20-08:00 summary: closes bpo-39605: Fix some casts to not cast away const. (GH-18453) gcc -Wcast-qual turns up a number of instances of casting away constness of pointers. Some of these can be safely modified, by either: Adding the const to the type cast, as in: - return _PyUnicode_FromUCS1((unsigned char*)s, size); + return _PyUnicode_FromUCS1((const unsigned char*)s, size); or, Removing the cast entirely, because it's not necessary (but probably was at one time), as in: - PyDTrace_FUNCTION_ENTRY((char *)filename, (char *)funcname, lineno); + PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); These changes will not change code, but they will make it much easier to check for errors in consts (cherry picked from commit e6be9b59a911626d6597fe148c32f0342bd2bd24) Co-authored-by: Andy Lester <andy(a)petdance.com> files: M Modules/_io/textio.c M Objects/bytes_methods.c M Objects/memoryobject.c M Objects/stringlib/asciilib.h M Objects/stringlib/codecs.h M Objects/stringlib/find_max_char.h M Objects/unicodeobject.c M Python/marshal.c M Python/pyhash.c diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 2f4a5245681f0..a4bf7cd107670 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1985,7 +1985,7 @@ find_control_char(int kind, const char *s, const char *end, Py_UCS4 ch) { if (kind == PyUnicode_1BYTE_KIND) { assert(ch < 256); - return (char *) memchr((void *) s, (char) ch, end - s); + return (char *) memchr((const void *) s, (char) ch, end - s); } for (;;) { while (PyUnicode_READ(kind, s, 0) > ch) @@ -2003,7 +2003,7 @@ _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, int kind, const char *start, const char *end, Py_ssize_t *consumed) { - Py_ssize_t len = ((char*)end - (char*)start)/kind; + Py_ssize_t len = (end - start)/kind; if (translated) { /* Newlines are already translated, only search for \n */ diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index 7d13184205922..db030be4fe756 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -12,7 +12,7 @@ PyObject* _Py_bytes_isspace(const char *cptr, Py_ssize_t len) { const unsigned char *p - = (unsigned char *) cptr; + = (const unsigned char *) cptr; const unsigned char *e; /* Shortcut for single character strings */ @@ -42,7 +42,7 @@ PyObject* _Py_bytes_isalpha(const char *cptr, Py_ssize_t len) { const unsigned char *p - = (unsigned char *) cptr; + = (const unsigned char *) cptr; const unsigned char *e; /* Shortcut for single character strings */ @@ -72,7 +72,7 @@ PyObject* _Py_bytes_isalnum(const char *cptr, Py_ssize_t len) { const unsigned char *p - = (unsigned char *) cptr; + = (const unsigned char *) cptr; const unsigned char *e; /* Shortcut for single character strings */ @@ -123,7 +123,7 @@ _Py_bytes_isascii(const char *cptr, Py_ssize_t len) /* Help allocation */ const char *_p = p; while (_p < aligned_end) { - unsigned long value = *(unsigned long *) _p; + unsigned long value = *(const unsigned long *) _p; if (value & ASCII_CHAR_MASK) { Py_RETURN_FALSE; } @@ -154,7 +154,7 @@ PyObject* _Py_bytes_isdigit(const char *cptr, Py_ssize_t len) { const unsigned char *p - = (unsigned char *) cptr; + = (const unsigned char *) cptr; const unsigned char *e; /* Shortcut for single character strings */ @@ -184,7 +184,7 @@ PyObject* _Py_bytes_islower(const char *cptr, Py_ssize_t len) { const unsigned char *p - = (unsigned char *) cptr; + = (const unsigned char *) cptr; const unsigned char *e; int cased; @@ -218,7 +218,7 @@ PyObject* _Py_bytes_isupper(const char *cptr, Py_ssize_t len) { const unsigned char *p - = (unsigned char *) cptr; + = (const unsigned char *) cptr; const unsigned char *e; int cased; @@ -254,7 +254,7 @@ PyObject* _Py_bytes_istitle(const char *cptr, Py_ssize_t len) { const unsigned char *p - = (unsigned char *) cptr; + = (const unsigned char *) cptr; const unsigned char *e; int cased, previous_is_cased; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index a873ac1ec1ea5..92050791b5d16 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1681,8 +1681,8 @@ unpack_single(const char *ptr, const char *fmt) switch (fmt[0]) { /* signed integers and fast path for 'B' */ - case 'B': uc = *((unsigned char *)ptr); goto convert_uc; - case 'b': ld = *((signed char *)ptr); goto convert_ld; + case 'B': uc = *((const unsigned char *)ptr); goto convert_uc; + case 'b': ld = *((const signed char *)ptr); goto convert_ld; case 'h': UNPACK_SINGLE(ld, ptr, short); goto convert_ld; case 'i': UNPACK_SINGLE(ld, ptr, int); goto convert_ld; case 'l': UNPACK_SINGLE(ld, ptr, long); goto convert_ld; @@ -2683,8 +2683,8 @@ unpack_cmp(const char *p, const char *q, char fmt, switch (fmt) { /* signed integers and fast path for 'B' */ - case 'B': return *((unsigned char *)p) == *((unsigned char *)q); - case 'b': return *((signed char *)p) == *((signed char *)q); + case 'B': return *((const unsigned char *)p) == *((const unsigned char *)q); + case 'b': return *((const signed char *)p) == *((const signed char *)q); case 'h': CMP_SINGLE(p, q, short); return equal; case 'i': CMP_SINGLE(p, q, int); return equal; case 'l': CMP_SINGLE(p, q, long); return equal; diff --git a/Objects/stringlib/asciilib.h b/Objects/stringlib/asciilib.h index d0fc18d22fa5f..8f83614488e84 100644 --- a/Objects/stringlib/asciilib.h +++ b/Objects/stringlib/asciilib.h @@ -18,7 +18,7 @@ #define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL #define STRINGLIB_STR PyUnicode_1BYTE_DATA #define STRINGLIB_LEN PyUnicode_GET_LENGTH -#define STRINGLIB_NEW(STR,LEN) _PyUnicode_FromASCII((char*)(STR),(LEN)) +#define STRINGLIB_NEW(STR,LEN) _PyUnicode_FromASCII((const char*)(STR),(LEN)) #define STRINGLIB_CHECK PyUnicode_Check #define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index d6f2b98f2b30a..269a5581f7005 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -46,7 +46,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, /* Read a whole long at a time (either 4 or 8 bytes), and do a fast unrolled copy if it only contains ASCII characters. */ - unsigned long value = *(unsigned long *) _s; + unsigned long value = *(const unsigned long *) _s; if (value & ASCII_CHAR_MASK) break; #if PY_LITTLE_ENDIAN @@ -515,7 +515,7 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, /* Fast path for runs of in-range non-surrogate chars. */ const unsigned char *_q = q; while (_q < aligned_end) { - unsigned long block = * (unsigned long *) _q; + unsigned long block = * (const unsigned long *) _q; if (native_ordering) { /* Can use buffer directly */ if (block & FAST_CHAR_MASK) diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h index 8ccbc3094463d..f4e0a7761d311 100644 --- a/Objects/stringlib/find_max_char.h +++ b/Objects/stringlib/find_max_char.h @@ -28,7 +28,7 @@ STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) /* Help register allocation */ const unsigned char *_p = p; while (_p < aligned_end) { - unsigned long value = *(unsigned long *) _p; + unsigned long value = *(const unsigned long *) _p; if (value & UCS1_ASCII_CHAR_MASK) return 255; _p += SIZEOF_LONG; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0556eff8d0713..8ba379e8887e9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -171,8 +171,8 @@ extern "C" { #define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ do { \ to_type *_to = (to_type *)(to); \ - const from_type *_iter = (from_type *)(begin); \ - const from_type *_end = (from_type *)(end); \ + const from_type *_iter = (const from_type *)(begin);\ + const from_type *_end = (const from_type *)(end);\ Py_ssize_t n = (_end) - (_iter); \ const from_type *_unrolled_end = \ _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ @@ -919,21 +919,21 @@ findchar(const void *s, int kind, if ((Py_UCS1) ch != ch) return -1; if (direction > 0) - return ucs1lib_find_char((Py_UCS1 *) s, size, (Py_UCS1) ch); + return ucs1lib_find_char((const Py_UCS1 *) s, size, (Py_UCS1) ch); else - return ucs1lib_rfind_char((Py_UCS1 *) s, size, (Py_UCS1) ch); + return ucs1lib_rfind_char((const Py_UCS1 *) s, size, (Py_UCS1) ch); case PyUnicode_2BYTE_KIND: if ((Py_UCS2) ch != ch) return -1; if (direction > 0) - return ucs2lib_find_char((Py_UCS2 *) s, size, (Py_UCS2) ch); + return ucs2lib_find_char((const Py_UCS2 *) s, size, (Py_UCS2) ch); else - return ucs2lib_rfind_char((Py_UCS2 *) s, size, (Py_UCS2) ch); + return ucs2lib_rfind_char((const Py_UCS2 *) s, size, (Py_UCS2) ch); case PyUnicode_4BYTE_KIND: if (direction > 0) - return ucs4lib_find_char((Py_UCS4 *) s, size, ch); + return ucs4lib_find_char((const Py_UCS4 *) s, size, ch); else - return ucs4lib_rfind_char((Py_UCS4 *) s, size, ch); + return ucs4lib_rfind_char((const Py_UCS4 *) s, size, ch); default: Py_UNREACHABLE(); } @@ -3353,7 +3353,7 @@ PyUnicode_Decode(const char *s, /* Decode via the codec registry */ buffer = NULL; - if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0) + if (PyBuffer_FillInfo(&info, NULL, (const void *)s, size, 1, PyBUF_FULL_RO) < 0) goto onError; buffer = PyMemoryView_FromBuffer(&info); if (buffer == NULL) @@ -4861,7 +4861,7 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) /* Help allocation */ const char *_p = p; while (_p < aligned_end) { - unsigned long value = *(unsigned long *) _p; + unsigned long value = *(const unsigned long *) _p; if (value & ASCII_CHAR_MASK) break; _p += SIZEOF_LONG; @@ -5406,7 +5406,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s, PyObject *errorHandler = NULL; PyObject *exc = NULL; - q = (unsigned char *)s; + q = (const unsigned char *)s; e = q + size; if (byteorder) @@ -5731,7 +5731,7 @@ PyUnicode_DecodeUTF16Stateful(const char *s, PyObject *exc = NULL; const char *encoding; - q = (unsigned char *)s; + q = (const unsigned char *)s; e = q + size; if (byteorder) @@ -6660,7 +6660,7 @@ PyUnicode_DecodeLatin1(const char *s, const char *errors) { /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ - return _PyUnicode_FromUCS1((unsigned char*)s, size); + return _PyUnicode_FromUCS1((const unsigned char*)s, size); } /* create or adjust a UnicodeEncodeError */ @@ -13708,7 +13708,7 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, if (len == -1) len = strlen(ascii); - assert(ucs1lib_find_max_char((Py_UCS1*)ascii, (Py_UCS1*)ascii + len) < 128); + assert(ucs1lib_find_max_char((const Py_UCS1*)ascii, (const Py_UCS1*)ascii + len) < 128); if (writer->buffer == NULL && !writer->overallocate) { PyObject *str; @@ -13767,7 +13767,7 @@ _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, { Py_UCS4 maxchar; - maxchar = ucs1lib_find_max_char((Py_UCS1*)str, (Py_UCS1*)str + len); + maxchar = ucs1lib_find_max_char((const Py_UCS1*)str, (const Py_UCS1*)str + len); if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1) return -1; unicode_write_cstr(writer->buffer, writer->pos, str, len); diff --git a/Python/marshal.c b/Python/marshal.c index c6a06e8c23976..d3fee32380b58 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -734,7 +734,7 @@ r_byte(RFILE *p) else { const char *ptr = r_string(1, p); if (ptr != NULL) - c = *(unsigned char *) ptr; + c = *(const unsigned char *) ptr; } return c; } diff --git a/Python/pyhash.c b/Python/pyhash.c index 4c0b929586fc1..ba224ee373631 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -366,7 +366,7 @@ static PyHash_FuncDef PyHash_Func = {fnv, "fnv", 8 * SIZEOF_PY_HASH_T, static uint64_t siphash24(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { uint64_t b = (uint64_t)src_sz << 56; - const uint8_t *in = (uint8_t*)src; + const uint8_t *in = (const uint8_t*)src; uint64_t v0 = k0 ^ 0x736f6d6570736575ULL; uint64_t v1 = k1 ^ 0x646f72616e646f6dULL;

1 0