Mailman 3 November 2022 - Python-checkins

GH-99877)
by iritkatriel 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/18a6967544795cdcce45b45700b7a9ed39… commit: 18a6967544795cdcce45b45700b7a9ed3994b8fb branch: main author: Irit Katriel <1055913+iritkatriel(a)users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel(a)users.noreply.github.com> date: 2022-11-30T16:16:54Z summary: GH-99877) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index da31f1c45c36..e200c5abb598 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -171,6 +171,24 @@ struct instr { struct basicblock_ *i_except; /* target block when exception is raised */ }; +/* One arg*/ +#define INSTR_SET_OP1(I, OP, ARG) \ + do { \ + assert(HAS_ARG(OP)); \ + struct instr *_instr__ptr_ = (I); \ + _instr__ptr_->i_opcode = (OP); \ + _instr__ptr_->i_oparg = (ARG); \ + } while (0); + +/* No args*/ +#define INSTR_SET_OP0(I, OP) \ + do { \ + assert(!HAS_ARG(OP)); \ + struct instr *_instr__ptr_ = (I); \ + _instr__ptr_->i_opcode = (OP); \ + _instr__ptr_->i_oparg = 0; \ + } while (0); + typedef struct exceptstack { struct basicblock_ *handlers[CO_MAXBLOCKS+1]; int depth; @@ -218,7 +236,8 @@ instr_size(struct instr *instruction) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); - int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; + int oparg = instruction->i_oparg; + assert(HAS_ARG(opcode) || oparg == 0); int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); int caches = _PyOpcode_Caches[opcode]; return extended_args + 1 + caches; @@ -229,7 +248,8 @@ write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); - int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; + int oparg = instruction->i_oparg; + assert(HAS_ARG(opcode) || oparg == 0); int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: @@ -7598,7 +7618,7 @@ convert_exception_handlers_to_nops(basicblock *entryblock) { for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { - instr->i_opcode = NOP; + INSTR_SET_OP0(instr, NOP); } } } @@ -8723,7 +8743,7 @@ remove_redundant_jumps(cfg_builder *g) { } if (last->i_target == b->b_next) { assert(b->b_next->b_iused); - last->i_opcode = NOP; + INSTR_SET_OP0(last, NOP); } } } @@ -8999,10 +9019,9 @@ fold_tuple_on_constants(PyObject *const_cache, } Py_DECREF(newconst); for (int i = 0; i < n; i++) { - inst[i].i_opcode = NOP; + INSTR_SET_OP0(&inst[i], NOP); } - inst[n].i_opcode = LOAD_CONST; - inst[n].i_oparg = (int)index; + INSTR_SET_OP1(&inst[n], LOAD_CONST, (int)index); return 0; } @@ -9099,7 +9118,7 @@ swaptimize(basicblock *block, int *ix) } // NOP out any unused instructions: while (0 <= current) { - instructions[current--].i_opcode = NOP; + INSTR_SET_OP0(&instructions[current--], NOP); } PyMem_Free(stack); *ix += len - 1; @@ -9165,7 +9184,7 @@ apply_static_swaps(basicblock *block, int i) } } // Success! - swap->i_opcode = NOP; + INSTR_SET_OP0(swap, NOP); struct instr temp = block->b_instr[j]; block->b_instr[j] = block->b_instr[k]; block->b_instr[k] = temp; @@ -9202,7 +9221,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); struct instr nop; - nop.i_opcode = NOP; + INSTR_SET_OP0(&nop, NOP); struct instr *target; for (int i = 0; i < bb->b_iused; i++) { struct instr *inst = &bb->b_instr[i]; @@ -9236,13 +9255,13 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) if (is_true == -1) { goto error; } - inst->i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); jump_if_true = nextop == POP_JUMP_IF_TRUE; if (is_true == jump_if_true) { bb->b_instr[i+1].i_opcode = JUMP; } else { - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); } break; case JUMP_IF_FALSE_OR_POP: @@ -9261,8 +9280,8 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) bb->b_instr[i+1].i_opcode = JUMP; } else { - inst->i_opcode = NOP; - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); } break; case IS_OP: @@ -9273,8 +9292,8 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) int jump_op = i+2 < bb->b_iused ? bb->b_instr[i+2].i_opcode : 0; if (Py_IsNone(cnt) && (jump_op == POP_JUMP_IF_FALSE || jump_op == POP_JUMP_IF_TRUE)) { unsigned char nextarg = bb->b_instr[i+1].i_oparg; - inst->i_opcode = NOP; - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); bb->b_instr[i+2].i_opcode = nextarg ^ (jump_op == POP_JUMP_IF_FALSE) ? POP_JUMP_IF_NOT_NONE : POP_JUMP_IF_NONE; } @@ -9292,12 +9311,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) if (nextop == UNPACK_SEQUENCE && oparg == bb->b_instr[i+1].i_oparg) { switch(oparg) { case 1: - inst->i_opcode = NOP; - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); continue; case 2: case 3: - inst->i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); bb->b_instr[i+1].i_opcode = SWAP; continue; } @@ -9406,7 +9425,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) break; case SWAP: if (oparg == 1) { - inst->i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); break; } if (swaptimize(bb, &i)) { @@ -9418,8 +9437,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) break; case PUSH_NULL: if (nextop == LOAD_GLOBAL && (inst[1].i_opcode & 1) == 0) { - inst->i_opcode = NOP; - inst->i_oparg = 0; + INSTR_SET_OP0(inst, NOP); inst[1].i_oparg |= 1; } break; @@ -9448,7 +9466,7 @@ inline_small_exit_blocks(basicblock *bb) { } basicblock *target = last->i_target; if (basicblock_exits_scope(target) && target->b_iused <= MAX_COPY_SIZE) { - last->i_opcode = NOP; + INSTR_SET_OP0(last, NOP); if (basicblock_append_instructions(bb, target) < 0) { return -1; }

1 0

Fix typo in exception message in `multiprocessing.pool` (#99900)
by kumaraditya303 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/a694b8222e8b0683682958222699953379… commit: a694b8222e8b0683682958222699953379fd2d48 branch: main author: Arne de Laat <arne(a)delaat.net> committer: kumaraditya303 <59607654+kumaraditya303(a)users.noreply.github.com> date: 2022-11-30T20:57:28+05:30 summary: Fix typo in exception message in `multiprocessing.pool` (#99900) files: M Lib/multiprocessing/pool.py diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 961d7e599184..4f5d88cb975c 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -696,7 +696,7 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, if (not result_handler.is_alive()) and (len(cache) != 0): raise AssertionError( - "Cannot have cache with result_hander not alive") + "Cannot have cache with result_handler not alive") result_handler._state = TERMINATE change_notifier.put(None)

1 0

gh-87092: move all localsplus preparation into separate function called from assembler stage (GH-99869)
by iritkatriel 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/ac12e3941f9c584c24b33d05beee95457e… commit: ac12e3941f9c584c24b33d05beee95457e9d06a3 branch: main author: Irit Katriel <1055913+iritkatriel(a)users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel(a)users.noreply.github.com> date: 2022-11-30T12:37:30Z summary: gh-87092: move all localsplus preparation into separate function called from assembler stage (GH-99869) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index dd8596defb8e..da31f1c45c36 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -8730,6 +8730,41 @@ remove_redundant_jumps(cfg_builder *g) { return 0; } +static int +prepare_localsplus(struct compiler* c, int code_flags) +{ + assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(c->u->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + assert(INT_MAX - nlocals - ncellvars > 0); + assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); + int nlocalsplus = nlocals + ncellvars + nfreevars; + int* cellfixedoffsets = build_cellfixedoffsets(c); + if (cellfixedoffsets == NULL) { + return -1; + } + + cfg_builder* g = CFG_BUILDER(c); + + // This must be called before fix_cell_offsets(). + if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + PyMem_Free(cellfixedoffsets); + return -1; + } + + int numdropped = fix_cell_offsets(c, g->g_entryblock, cellfixedoffsets); + PyMem_Free(cellfixedoffsets); // At this point we're done with it. + cellfixedoffsets = NULL; + if (numdropped < 0) { + return -1; + } + nlocalsplus -= numdropped; + return nlocalsplus; +} + static PyCodeObject * assemble(struct compiler *c, int addNone) { @@ -8751,20 +8786,6 @@ assemble(struct compiler *c, int addNone) ADDOP(c, NO_LOCATION, RETURN_VALUE); } - assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); - assert(INT_MAX - nlocals - ncellvars > 0); - assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); - int nlocalsplus = nlocals + ncellvars + nfreevars; - int *cellfixedoffsets = build_cellfixedoffsets(c); - if (cellfixedoffsets == NULL) { - goto error; - } - int nblocks = 0; for (basicblock *b = CFG_BUILDER(c)->g_block_list; b != NULL; b = b->b_list) { nblocks++; @@ -8787,19 +8808,6 @@ assemble(struct compiler *c, int addNone) } } - // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { - goto error; - } - - int numdropped = fix_cell_offsets(c, g->g_entryblock, cellfixedoffsets); - PyMem_Free(cellfixedoffsets); // At this point we're done with it. - cellfixedoffsets = NULL; - if (numdropped < 0) { - goto error; - } - nlocalsplus -= numdropped; - /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ if (translate_jump_labels_to_targets(g->g_entryblock)) { @@ -8839,6 +8847,12 @@ assemble(struct compiler *c, int addNone) } /** Assembly **/ + + int nlocalsplus = prepare_localsplus(c, code_flags); + if (nlocalsplus < 0) { + goto error; + } + int maxdepth = stackdepth(g->g_entryblock, code_flags); if (maxdepth < 0) { goto error; @@ -8904,9 +8918,6 @@ assemble(struct compiler *c, int addNone) error: Py_XDECREF(consts); assemble_free(&a); - if (cellfixedoffsets != NULL) { - PyMem_Free(cellfixedoffsets); - } return co; }

1 0

gh-99891: Fix infinite recursion in the tokenizer when showing warnings (GH-99893)
by miss-islington 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/417206a05c4545bde96c2bbbea92b53e6c… commit: 417206a05c4545bde96c2bbbea92b53e6cac0d48 branch: main author: Pablo Galindo Salgado <Pablogsal(a)gmail.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-11-30T03:36:06-08:00 summary: gh-99891: Fix infinite recursion in the tokenizer when showing warnings (GH-99893) Automerge-Triggered-By: GH:pablogsal files: A Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst M Lib/test/test_source_encoding.py M Parser/tokenizer.c M Parser/tokenizer.h diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index cfc4b13f18f3..b05173ad00d4 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -160,6 +160,18 @@ def test_file_parse_error_multiline(self): finally: os.unlink(TESTFN) + def test_tokenizer_fstring_warning_in_first_line(self): + source = "0b1and 2" + with open(TESTFN, "w") as fd: + fd.write("{}".format(source)) + try: + retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN) + self.assertIn(b"SyntaxWarning: invalid binary litera", stderr) + self.assertEqual(stderr.count(source.encode()), 1) + finally: + os.unlink(TESTFN) + + class AbstractSourceEncodingTest: def test_default_coding(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst new file mode 100644 index 000000000000..20cd361affea --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst @@ -0,0 +1,3 @@ +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index ce72e1529024..463c0e00ca14 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -97,6 +97,7 @@ tok_new(void) tok->async_def_nl = 0; tok->interactive_underflow = IUNDERFLOW_NORMAL; tok->str = NULL; + tok->report_warnings = 1; #ifdef Py_DEBUG tok->debug = _Py_GetConfig()->parser_debug; #endif @@ -1201,6 +1202,10 @@ indenterror(struct tok_state *tok) static int parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { + if (!tok->report_warnings) { + return 0; + } + PyObject *errmsg; va_list vargs; va_start(vargs, format); @@ -2239,6 +2244,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) } } struct token token; + // We don't want to report warnings here because it could cause infinite recursion + // if fetching the encoding shows a warning. + tok->report_warnings = 0; while (tok->lineno < 2 && tok->done == E_OK) { _PyTokenizer_Get(tok, &token); } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 2542d30e1da0..16a94d5f51d6 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -92,6 +92,7 @@ struct tok_state { NEWLINE token after it. */ /* How to proceed when asked for a new token in interactive mode */ enum interactive_underflow_t interactive_underflow; + int report_warnings; #ifdef Py_DEBUG int debug; #endif

1 0

gh-99824: Document that sqlite3.connect implicitly open a transaction if autocommit=False (#99825)
by erlend-aasland 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/19c38801ba2f42a220adece5d5f12e833b… commit: 19c38801ba2f42a220adece5d5f12e833b41822a branch: main author: Géry Ogam <gery.ogam(a)gmail.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2022-11-30T12:33:50+01:00 summary: gh-99824: Document that sqlite3.connect implicitly open a transaction if autocommit=False (#99825) Authored-by: C.A.M. Gerlach <CAM.Gerlach(a)Gerlach.CAM> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 7e2235b285b8..a9ee1e207d0c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2468,9 +2468,9 @@ which implies :pep:`249`-compliant transaction control. This means: * :mod:`!sqlite3` ensures that a transaction is always open, - so :meth:`Connection.commit` and :meth:`Connection.rollback` - will implicitly open a new transaction immediately after closing - the pending one. + so :func:`connect`, :meth:`Connection.commit`, and :meth:`Connection.rollback` + will implicitly open a new transaction + (immediately after closing the pending one, for the latter two). :mod:`!sqlite3` uses ``BEGIN DEFERRED`` statements when opening transactions. * Transactions should be committed explicitly using :meth:`!commit`. * Transactions should be rolled back explicitly using :meth:`!rollback`.

1 0

GH-81057: remove static state from suggestions.c (#99411)
by kumaraditya303 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/fe17d353134748dc772f8743ceadc2dd9e… commit: fe17d353134748dc772f8743ceadc2dd9e0db187 branch: main author: Kumar Aditya <59607654+kumaraditya303(a)users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303(a)users.noreply.github.com> date: 2022-11-30T16:55:16+05:30 summary: GH-81057: remove static state from suggestions.c (#99411) files: M Python/suggestions.c diff --git a/Python/suggestions.c b/Python/suggestions.c index ad645c7d96fa..f2c018ef2c45 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -41,10 +41,8 @@ substitution_cost(char a, char b) static Py_ssize_t levenshtein_distance(const char *a, size_t a_size, const char *b, size_t b_size, - size_t max_cost) + size_t max_cost, size_t *buffer) { - static size_t buffer[MAX_STRING_SIZE]; - // Both strings are the same (by identity) if (a == b) { return 0; @@ -147,12 +145,16 @@ calculate_suggestions(PyObject *dir, if (name_str == NULL) { return NULL; } - + size_t *buffer = PyMem_New(size_t, MAX_STRING_SIZE); + if (buffer == NULL) { + return PyErr_NoMemory(); + } for (int i = 0; i < dir_size; ++i) { PyObject *item = PyList_GET_ITEM(dir, i); Py_ssize_t item_size; const char *item_str = PyUnicode_AsUTF8AndSize(item, &item_size); if (item_str == NULL) { + PyMem_Free(buffer); return NULL; } if (PyUnicode_CompareWithASCIIString(name, item_str) == 0) { @@ -163,8 +165,8 @@ calculate_suggestions(PyObject *dir, // Don't take matches we've already beaten. max_distance = Py_MIN(max_distance, suggestion_distance - 1); Py_ssize_t current_distance = - levenshtein_distance(name_str, name_size, - item_str, item_size, max_distance); + levenshtein_distance(name_str, name_size, item_str, + item_size, max_distance, buffer); if (current_distance > max_distance) { continue; } @@ -173,6 +175,7 @@ calculate_suggestions(PyObject *dir, suggestion_distance = current_distance; } } + PyMem_Free(buffer); return Py_XNewRef(suggestion); } @@ -238,7 +241,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) if (!self) { goto error; } - + if (PyObject_HasAttr(self, name)) { Py_DECREF(dir); return PyUnicode_FromFormat("self.%S", name); @@ -401,6 +404,14 @@ _Py_UTF8_Edit_Cost(PyObject *a, PyObject *b, Py_ssize_t max_cost) if (max_cost == -1) { max_cost = MOVE_COST * Py_MAX(size_a, size_b); } - return levenshtein_distance(utf8_a, size_a, utf8_b, size_b, max_cost); + size_t *buffer = PyMem_New(size_t, MAX_STRING_SIZE); + if (buffer == NULL) { + PyErr_NoMemory(); + return -1; + } + Py_ssize_t res = levenshtein_distance(utf8_a, size_a, + utf8_b, size_b, max_cost, buffer); + PyMem_Free(buffer); + return res; }

1 0

Improve zip64 limit error message (#95892)
by kumaraditya303 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/59665d0280c2299ea87e9af45cedc90656… commit: 59665d0280c2299ea87e9af45cedc90656cb6f55 branch: main author: dmjohnsson23 <dmjohn235(a)gmail.com> committer: kumaraditya303 <59607654+kumaraditya303(a)users.noreply.github.com> date: 2022-11-30T16:44:41+05:30 summary: Improve zip64 limit error message (#95892) files: M Lib/zipfile/__init__.py diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 8f834267b28c..e1833dd1772d 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1202,10 +1202,10 @@ def close(self): if not self._zip64: if self._file_size > ZIP64_LIMIT: raise RuntimeError( - 'File size unexpectedly exceeded ZIP64 limit') + 'File size too large, try using force_zip64') if self._compress_size > ZIP64_LIMIT: raise RuntimeError( - 'Compressed size unexpectedly exceeded ZIP64 limit') + 'Compressed size too large, try using force_zip64') # Seek backwards and write file header (which will now include # correct CRC and file sizes)

1 0

gh-98253: Break potential reference cycles in external code worsened by typing.py lru_cache (#98591)
by AlexWaygood 30 Nov '22

30 Nov '22

https://github.com/python/cpython/commit/c314198fadd3dc79ee15f375a409163d8f… commit: c314198fadd3dc79ee15f375a409163d8fb33586 branch: main author: Wenzel Jakob <wenzel.jakob(a)epfl.ch> committer: AlexWaygood <Alex.Waygood(a)Gmail.com> date: 2022-11-30T08:33:32Z summary: gh-98253: Break potential reference cycles in external code worsened by typing.py lru_cache (#98591) files: A Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 127e3c942d04..38e227e3c55d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -325,6 +325,7 @@ def _flatten_literal_params(parameters): _cleanups = [] +_caches = {} def _tp_cache(func=None, /, *, typed=False): @@ -332,13 +333,20 @@ def _tp_cache(func=None, /, *, typed=False): original function for non-hashable arguments. """ def decorator(func): - cached = functools.lru_cache(typed=typed)(func) - _cleanups.append(cached.cache_clear) + # The callback 'inner' references the newly created lru_cache + # indirectly by performing a lookup in the global '_caches' dictionary. + # This breaks a reference that can be problematic when combined with + # C API extensions that leak references to types. See GH-98253. + + cache = functools.lru_cache(typed=typed)(func) + _caches[func] = cache + _cleanups.append(cache.cache_clear) + del cache @functools.wraps(func) def inner(*args, **kwds): try: - return cached(*args, **kwds) + return _caches[func](*args, **kwds) except TypeError: pass # All real errors (not unhashable args) are raised below. return func(*args, **kwds) diff --git a/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst b/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst new file mode 100644 index 000000000000..00df0070f3b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst @@ -0,0 +1,10 @@ +The implementation of the typing module is now more resilient to reference +leaks in binary extension modules. + +Previously, a reference leak in a typed C API-based extension module could leak +internals of the typing module, which could in turn introduce leaks in +essentially any other package with typed function signatures. Although the +typing package is not the original source of the problem, such non-local +dependences exacerbate debugging of large-scale projects, and the +implementation was therefore changed to reduce harm by providing better +isolation.

1 0

gh-99127: Allow some features of syslog to the main interpreter only (gh-99128)
by corona10 29 Nov '22

29 Nov '22

https://github.com/python/cpython/commit/8bb2303fd75c299d8fc85229889ac75e86… commit: 8bb2303fd75c299d8fc85229889ac75e867c135c branch: main author: Dong-hee Na <donghee.na(a)python.org> committer: corona10 <donghee.na92(a)gmail.com> date: 2022-11-30T07:58:20+09:00 summary: gh-99127: Allow some features of syslog to the main interpreter only (gh-99128) files: A Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst M Doc/library/syslog.rst M Doc/whatsnew/3.12.rst M Lib/test/test_syslog.py M Modules/syslogmodule.c diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 766ff57cc66d..f29ef03267b1 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -40,6 +40,13 @@ The module defines the following functions: it wasn't called prior to the call to :func:`syslog`, deferring to the syslog implementation to call ``openlog()``. + .. versionchanged:: 3.12 + This function is restricted in subinterpreters. + (Only code that runs in multiple interpreters is affected and + the restriction is not relevant for most users.) + :func:`openlog` must be called in the main interpreter before :func:`syslog` may be used + in a subinterpreter. Otherwise it will raise :exc:`RuntimeError`. + .. function:: openlog([ident[, logoption[, facility]]]) @@ -60,6 +67,13 @@ The module defines the following functions: In previous versions, keyword arguments were not allowed, and *ident* was required. + .. versionchanged:: 3.12 + This function is restricted in subinterpreters. + (Only code that runs in multiple interpreters is affected and + the restriction is not relevant for most users.) + This may only be called in the main interpreter. + It will raise :exc:`RuntimeError` if called in a subinterpreter. + .. function:: closelog() @@ -72,6 +86,13 @@ The module defines the following functions: .. audit-event:: syslog.closelog "" syslog.closelog + .. versionchanged:: 3.12 + This function is restricted in subinterpreters. + (Only code that runs in multiple interpreters is affected and + the restriction is not relevant for most users.) + This may only be called in the main interpreter. + It will raise :exc:`RuntimeError` if called in a subinterpreter. + .. function:: setlogmask(maskpri) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index dff4de621b4c..c0f98b59ccaf 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -673,6 +673,15 @@ Changes in the Python API :class:`bytes` type is accepted for bytes strings. (Contributed by Victor Stinner in :gh:`98393`.) +* :func:`syslog.openlog` and :func:`syslog.closelog` now fail if used in subinterpreters. + :func:`syslog.syslog` may still be used in subinterpreters, + but now only if :func:`syslog.openlog` has already been called in the main interpreter. + These new restrictions do not apply to the main interpreter, + so only a very small set of users might be affected. + This change helps with interpreter isolation. Furthermore, :mod:`syslog` is a wrapper + around process-global resources, which are best managed from the main interpreter. + (Contributed by Dong-hee Na in :gh:`99127`.) + Build Changes ============= diff --git a/Lib/test/test_syslog.py b/Lib/test/test_syslog.py index 2125ec58d87e..54db80fa9df1 100644 --- a/Lib/test/test_syslog.py +++ b/Lib/test/test_syslog.py @@ -5,6 +5,7 @@ import threading import time import unittest +from textwrap import dedent # XXX(nnorwitz): This test sucks. I don't know of a platform independent way # to verify that the messages were really logged. @@ -78,6 +79,69 @@ def logger(): finally: sys.setswitchinterval(orig_si) + def test_subinterpreter_syslog(self): + # syslog.syslog() is not allowed in subinterpreters, but only if + # syslog.openlog() hasn't been called in the main interpreter yet. + with self.subTest('before openlog()'): + code = dedent(''' + import syslog + caught_error = False + try: + syslog.syslog('foo') + except RuntimeError: + caught_error = True + assert(caught_error) + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + + syslog.openlog() + try: + with self.subTest('after openlog()'): + code = dedent(''' + import syslog + syslog.syslog('foo') + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + finally: + syslog.closelog() + + def test_subinterpreter_openlog(self): + try: + code = dedent(''' + import syslog + caught_error = False + try: + syslog.openlog() + except RuntimeError: + caught_error = True + + assert(caught_error) + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + finally: + syslog.closelog() + + def test_subinterpreter_closelog(self): + syslog.openlog('python') + try: + code = dedent(''' + import syslog + caught_error = False + try: + syslog.closelog() + except RuntimeError: + caught_error = True + + assert(caught_error) + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + finally: + syslog.closelog() + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst new file mode 100644 index 000000000000..e93ae4e7b127 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst @@ -0,0 +1 @@ +Allow some features of :mod:`syslog` to the main interpreter only. Patch by Dong-hee Na. diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 23833b728503..f45aa5227f1c 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -61,10 +61,16 @@ module syslog #include "clinic/syslogmodule.c.h" -/* only one instance, only one syslog, so globals should be ok */ -static PyObject *S_ident_o = NULL; /* identifier, held by openlog() */ +/* only one instance, only one syslog, so globals should be ok, + * these fields are writable from the main interpreter only. */ +static PyObject *S_ident_o = NULL; // identifier, held by openlog() static char S_log_open = 0; +static inline int +is_main_interpreter(void) +{ + return (PyInterpreterState_Get() == PyInterpreterState_Main()); +} static PyObject * syslog_get_argv(void) @@ -135,6 +141,13 @@ syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, long facility) /*[clinic end generated code: output=5476c12829b6eb75 input=8a987a96a586eee7]*/ { + // Since the sys.openlog changes the process level state of syslog library, + // this operation is only allowed for the main interpreter. + if (!is_main_interpreter()) { + PyErr_SetString(PyExc_RuntimeError, "subinterpreter can't use syslog.openlog()"); + return NULL; + } + const char *ident_str = NULL; if (ident) { @@ -195,6 +208,11 @@ syslog_syslog_impl(PyObject *module, int group_left_1, int priority, /* if log is not opened, open it now */ if (!S_log_open) { + if (!is_main_interpreter()) { + PyErr_SetString(PyExc_RuntimeError, "subinterpreter can't use syslog.syslog() " + "until the syslog is opened by the main interpreter"); + return NULL; + } PyObject *openlog_ret = syslog_openlog_impl(module, NULL, 0, LOG_USER); if (openlog_ret == NULL) { return NULL; @@ -229,6 +247,13 @@ static PyObject * syslog_closelog_impl(PyObject *module) /*[clinic end generated code: output=97890a80a24b1b84 input=fb77a54d447acf07]*/ { + // Since the sys.closelog changes the process level state of syslog library, + // this operation is only allowed for the main interpreter. + if (!is_main_interpreter()) { + PyErr_SetString(PyExc_RuntimeError, "sunbinterpreter can't use syslog.closelog()"); + return NULL; + } + if (PySys_Audit("syslog.closelog", NULL) < 0) { return NULL; }

1 0

gh-82836: fix private network check (GH-97733)
by miss-islington 29 Nov '22

29 Nov '22

https://github.com/python/cpython/commit/8dafefcace5d30aa6bb319bf3f0723864d… commit: 8dafefcace5d30aa6bb319bf3f0723864d94201c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-11-29T14:12:57-08:00 summary: gh-82836: fix private network check (GH-97733) Fixes private checks for network objects. The previous method would incorrectly return True for a private check in cases such as "0.0.0.0/0". (cherry picked from commit ed391090cc8332406e6225d40877db6ff44a7104) Co-authored-by: Pete Wicken <2273100+JamoBox(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst M Lib/ipaddress.py M Lib/test/test_ipaddress.py diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 3f15601e700d..1cb71d8032e1 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1077,15 +1077,16 @@ def is_link_local(self): @property def is_private(self): - """Test if this address is allocated for private networks. + """Test if this network belongs to a private range. Returns: - A boolean, True if the address is reserved per + A boolean, True if the network is reserved per iana-ipv4-special-registry or iana-ipv6-special-registry. """ - return (self.network_address.is_private and - self.broadcast_address.is_private) + return any(self.network_address in priv_network and + self.broadcast_address in priv_network + for priv_network in self._constants._private_networks) @property def is_global(self): @@ -1122,6 +1123,15 @@ def is_loopback(self): return (self.network_address.is_loopback and self.broadcast_address.is_loopback) + +class _BaseConstants: + + _private_networks = [] + + +_BaseNetwork._constants = _BaseConstants + + class _BaseV4: """Base IPv4 object. @@ -1561,6 +1571,7 @@ class _IPv4Constants: IPv4Address._constants = _IPv4Constants +IPv4Network._constants = _IPv4Constants class _BaseV6: @@ -2285,3 +2296,4 @@ class _IPv6Constants: IPv6Address._constants = _IPv6Constants +IPv6Network._constants = _IPv6Constants diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 5c656c49e2e7..a5388b2e5deb 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2277,6 +2277,39 @@ def testReservedIpv4(self): self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback) self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) + def testPrivateNetworks(self): + self.assertEqual(False, ipaddress.ip_network("0.0.0.0/0").is_private) + self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) + + self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private) + self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private) + self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private) + + self.assertEqual(False, ipaddress.ip_network("::/0").is_private) + self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private) + + self.assertEqual(True, ipaddress.ip_network("::1/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) + self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) + self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) + self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) + self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private) + def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast) diff --git a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst new file mode 100644 index 000000000000..dcbea66d66bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst @@ -0,0 +1 @@ +Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would return True from this method; now they correctly return False.

1 0