Mailman 3 September 2019 - Python-checkins

bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079) (GH-16464)
by Gregory P. Smith Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/36c6fa968016a46a39c3cdbd0a17ea5490… commit: 36c6fa968016a46a39c3cdbd0a17ea5490dfa343 branch: 3.8 author: Gregory P. Smith <greg(a)krypto.org> committer: GitHub <noreply(a)github.com> date: 2019-09-28T08:22:00-07:00 summary: bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079) (GH-16464) Document that lnotab can contain invalid bytecode offsets (because of terrible reasons that are difficult to fix). Make dis.findlinestarts() ignore invalid offsets in lnotab. All other uses of lnotab in CPython (various reimplementations of addr2line or line2addr in Python, C and gdb) already ignore this, because they take an address to look for, instead. Add tests for the result of dis.findlinestarts() on wacky constructs in test_peepholer.py, because it's the easiest place to add them. (cherry picked from commit c8165036f374cd2ee64d4314eeb2514f7acb5026) files: A Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst M Lib/dis.py M Lib/test/test_peepholer.py M Objects/lnotab_notes.txt diff --git a/Lib/dis.py b/Lib/dis.py index a25fb2b41764..10e5f7fb08ab 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -454,6 +454,7 @@ def findlinestarts(code): """ byte_increments = code.co_lnotab[0::2] line_increments = code.co_lnotab[1::2] + bytecode_len = len(code.co_code) lastlineno = None lineno = code.co_firstlineno @@ -464,6 +465,10 @@ def findlinestarts(code): yield (addr, lineno) lastlineno = lineno addr += byte_incr + if addr >= bytecode_len: + # The rest of the lnotab byte offsets are past the end of + # the bytecode, so the lines were optimized away. + return if line_incr >= 0x80: # line_increments is an array of 8-bit signed integers line_incr -= 0x100 diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 9c206d1d09c0..01c00369156b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -40,6 +40,20 @@ def check_jump_targets(self, code): self.fail(f'{instr.opname} at {instr.offset} ' f'jumps to {tgt.opname} at {tgt.offset}') + def check_lnotab(self, code): + "Check that the lnotab byte offsets are sensible." + code = dis._get_code_object(code) + lnotab = list(dis.findlinestarts(code)) + # Don't bother checking if the line info is sensible, because + # most of the line info we can get at comes from lnotab. + min_bytecode = min(t[0] for t in lnotab) + max_bytecode = max(t[0] for t in lnotab) + self.assertGreaterEqual(min_bytecode, 0) + self.assertLess(max_bytecode, len(code.co_code)) + # This could conceivably test more (and probably should, as there + # aren't very many tests of lnotab), if peepholer wasn't scheduled + # to be replaced anyway. + def test_unot(self): # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' def unot(x): @@ -48,6 +62,7 @@ def unot(x): self.assertNotInBytecode(unot, 'UNARY_NOT') self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') + self.check_lnotab(unot) def test_elim_inversion_of_is_or_in(self): for line, cmp_op in ( @@ -58,6 +73,7 @@ def test_elim_inversion_of_is_or_in(self): ): code = compile(line, '', 'single') self.assertInBytecode(code, 'COMPARE_OP', cmp_op) + self.check_lnotab(code) def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False @@ -75,6 +91,7 @@ def h(): for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + self.check_lnotab(func) def f(): 'Adding a docstring made this test fail in Py2.5.0' @@ -82,6 +99,7 @@ def f(): self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) + self.check_lnotab(f) def test_while_one(self): # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx @@ -93,6 +111,7 @@ def f(): self.assertNotInBytecode(f, elem) for elem in ('JUMP_ABSOLUTE',): self.assertInBytecode(f, elem) + self.check_lnotab(f) def test_pack_unpack(self): for line, elem in ( @@ -104,6 +123,7 @@ def test_pack_unpack(self): self.assertInBytecode(code, elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') self.assertNotInBytecode(code, 'UNPACK_TUPLE') + self.check_lnotab(code) def test_folding_of_tuples_of_constants(self): for line, elem in ( @@ -116,6 +136,7 @@ def test_folding_of_tuples_of_constants(self): code = compile(line,'','single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.check_lnotab(code) # Long tuples should be folded too. code = compile(repr(tuple(range(10000))),'','single') @@ -124,6 +145,7 @@ def test_folding_of_tuples_of_constants(self): load_consts = [instr for instr in dis.get_instructions(code) if instr.opname == 'LOAD_CONST'] self.assertEqual(len(load_consts), 2) + self.check_lnotab(code) # Bug 1053819: Tuple of constants misidentified when presented with: # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . @@ -141,6 +163,7 @@ def crater(): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ],) + self.check_lnotab(crater) def test_folding_of_lists_of_constants(self): for line, elem in ( @@ -153,6 +176,7 @@ def test_folding_of_lists_of_constants(self): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_LIST') + self.check_lnotab(code) def test_folding_of_sets_of_constants(self): for line, elem in ( @@ -166,6 +190,7 @@ def test_folding_of_sets_of_constants(self): code = compile(line, '', 'single') self.assertNotInBytecode(code, 'BUILD_SET') self.assertInBytecode(code, 'LOAD_CONST', elem) + self.check_lnotab(code) # Ensure that the resulting code actually works: def f(a): @@ -176,9 +201,11 @@ def g(a): self.assertTrue(f(3)) self.assertTrue(not f(4)) + self.check_lnotab(f) self.assertTrue(not g(3)) self.assertTrue(g(4)) + self.check_lnotab(g) def test_folding_of_binops_on_constants(self): @@ -203,41 +230,50 @@ def test_folding_of_binops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('BINARY_')) + self.check_lnotab(code) # Verify that unfoldables are skipped code = compile('a=2+"b"', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 2) self.assertInBytecode(code, 'LOAD_CONST', 'b') + self.check_lnotab(code) # Verify that large sequences do not result from folding code = compile('a="x"*10000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 10000) self.assertNotIn("x"*10000, code.co_consts) + self.check_lnotab(code) code = compile('a=1<<1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(1<<1000, code.co_consts) + self.check_lnotab(code) code = compile('a=2**1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(2**1000, code.co_consts) + self.check_lnotab(code) def test_binary_subscr_on_unicode(self): # valid code get optimized code = compile('"foo"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 'f') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) code = compile('"\u0061\uffff"[1]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\uffff') self.assertNotInBytecode(code,'BINARY_SUBSCR') + self.check_lnotab(code) # With PEP 393, non-BMP char get optimized code = compile('"\U00012345"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) # invalid code doesn't get optimized # out of range code = compile('"fuu"[10]', '', 'single') self.assertInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) def test_folding_of_unaryops_on_constants(self): for line, elem in ( @@ -252,13 +288,15 @@ def test_folding_of_unaryops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(code) # Check that -0.0 works after marshaling def negzero(): return -(1.0-1.0) - for instr in dis.get_instructions(code): + for instr in dis.get_instructions(negzero): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(negzero) # Verify that unfoldables are skipped for line, elem, opname in ( @@ -268,6 +306,7 @@ def negzero(): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertInBytecode(code, opname) + self.check_lnotab(code) def test_elim_extra_return(self): # RETURN LOAD_CONST None RETURN --> RETURN @@ -277,6 +316,7 @@ def f(x): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 1) + self.check_lnotab(f) def test_elim_jump_to_return(self): # JUMP_FORWARD to RETURN --> RETURN @@ -290,6 +330,7 @@ def f(cond, true_value, false_value): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 2) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump(self): # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump @@ -302,6 +343,7 @@ def f(): else: baz() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump2(self): # POP_JUMP_IF_FALSE to JUMP_ABSOLUTE --> POP_JUMP_IF_FALSE to non-jump @@ -312,6 +354,7 @@ def f(): or d): a = foo() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump3(self): # Intentionally use two-line expressions to test issue37213. @@ -320,18 +363,21 @@ def f(a, b, c): return ((a and b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_FALSE_OR_POP'), 2) # JUMP_IF_TRUE_OR_POP to JUMP_IF_TRUE_OR_POP --> JUMP_IF_TRUE_OR_POP to non-jump def f(a, b, c): return ((a or b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_TRUE_OR_POP'), 2) # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump def f(a, b, c): return ((a and b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_FALSE') @@ -340,6 +386,7 @@ def f(a, b, c): return ((a or b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_TRUE') @@ -360,6 +407,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 6) + self.check_lnotab(f) def test_elim_jump_after_return2(self): # Eliminate dead code: jumps immediately after returns can't be reached @@ -374,6 +422,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 2) + self.check_lnotab(f) def test_make_function_doesnt_bail(self): def f(): @@ -381,6 +430,7 @@ def g()->1+1: pass return g self.assertNotInBytecode(f, 'BINARY_ADD') + self.check_lnotab(f) def test_constant_folding(self): # Issue #11244: aggressive constant folding. @@ -401,17 +451,20 @@ def test_constant_folding(self): self.assertFalse(instr.opname.startswith('UNARY_')) self.assertFalse(instr.opname.startswith('BINARY_')) self.assertFalse(instr.opname.startswith('BUILD_')) + self.check_lnotab(code) def test_in_literal_list(self): def containtest(): return x in [a, b] self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + self.check_lnotab(containtest) def test_iterate_literal_list(self): def forloop(): for x in [a, b]: pass self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + self.check_lnotab(forloop) def test_condition_with_binop_with_bools(self): def f(): @@ -419,6 +472,28 @@ def f(): return 1 return 0 self.assertEqual(f(), 1) + self.check_lnotab(f) + + def test_if_with_if_expression(self): # XXX does this belong in 3.8? + # Check bpo-37289 + def f(x): + if (True if x else False): + return True + return False + self.assertTrue(f(True)) + self.check_lnotab(f) + + def test_trailing_nops(self): + # Check the lnotab of a function that even after trivial + # optimization has trailing nops, which the lnotab adjustment has to + # handle properly (bpo-38115). + def f(x): + while 1: + return 3 + while 1: + return 5 + return 6 + self.check_lnotab(f) class TestBuglets(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst new file mode 100644 index 000000000000..5119c0546e36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst @@ -0,0 +1 @@ +Fix a bug in dis.findlinestarts() where it would return invalid bytecode offsets. Document that a code object's co_lnotab can contain invalid bytecode offsets. \ No newline at end of file diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 3dab2b986616..71a297971828 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -3,7 +3,9 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes disguised as a Python bytes object. It is used to map bytecode offsets to source code line #s for tracebacks and to identify line number boundaries for -line tracing. +line tracing. Because of internals of the peephole optimizer, it's possible +for lnotab to contain bytecode offsets that are no longer valid (for example +if the optimizer removed the last line in a function). The array is conceptually a compressed list of (bytecode offset increment, line number increment)

1 0

bpo-37408: Precise that Tarfile "format" argument only concerns writing. (GH-14389)
by Gregory P. Smith Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/c5a7e0ce194c0eafe82eb3e43188101239… commit: c5a7e0ce194c0eafe82eb3e431881012398e7d46 branch: master author: Pascal Chambon <pythoniks(a)gmail.com> committer: Gregory P. Smith <greg(a)krypto.org> date: 2019-09-28T08:04:44-07:00 summary: bpo-37408: Precise that Tarfile "format" argument only concerns writing. (GH-14389) files: M Doc/library/tarfile.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f25af8ca6a33..c34f2c4a5702 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -290,9 +290,10 @@ be finalized; only the internally used file object will be closed. See the *fileobj* is not closed, when :class:`TarFile` is closed. - *format* controls the archive format. It must be one of the constants + *format* controls the archive format for writing. It must be one of the constants :const:`USTAR_FORMAT`, :const:`GNU_FORMAT` or :const:`PAX_FORMAT` that are - defined at module level. + defined at module level. When reading, format will be automatically detected, even + if different formats are present in a single archive. The *tarinfo* argument can be used to replace the default :class:`TarInfo` class with a different one.

1 0

bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079)
by Gregory P. Smith Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/c8165036f374cd2ee64d4314eeb2514f7a… commit: c8165036f374cd2ee64d4314eeb2514f7acb5026 branch: master author: T. Wouters <thomas(a)python.org> committer: Gregory P. Smith <greg(a)krypto.org> date: 2019-09-28T07:49:15-07:00 summary: bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079) Document that lnotab can contain invalid bytecode offsets (because of terrible reasons that are difficult to fix). Make dis.findlinestarts() ignore invalid offsets in lnotab. All other uses of lnotab in CPython (various reimplementations of addr2line or line2addr in Python, C and gdb) already ignore this, because they take an address to look for, instead. Add tests for the result of dis.findlinestarts() on wacky constructs in test_peepholer.py, because it's the easiest place to add them. files: A Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst M Lib/dis.py M Lib/test/test_peepholer.py M Objects/lnotab_notes.txt diff --git a/Lib/dis.py b/Lib/dis.py index a25fb2b41764..10e5f7fb08ab 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -454,6 +454,7 @@ def findlinestarts(code): """ byte_increments = code.co_lnotab[0::2] line_increments = code.co_lnotab[1::2] + bytecode_len = len(code.co_code) lastlineno = None lineno = code.co_firstlineno @@ -464,6 +465,10 @@ def findlinestarts(code): yield (addr, lineno) lastlineno = lineno addr += byte_incr + if addr >= bytecode_len: + # The rest of the lnotab byte offsets are past the end of + # the bytecode, so the lines were optimized away. + return if line_incr >= 0x80: # line_increments is an array of 8-bit signed integers line_incr -= 0x100 diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 47dee33076c5..23cc36c60537 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -40,6 +40,20 @@ def check_jump_targets(self, code): self.fail(f'{instr.opname} at {instr.offset} ' f'jumps to {tgt.opname} at {tgt.offset}') + def check_lnotab(self, code): + "Check that the lnotab byte offsets are sensible." + code = dis._get_code_object(code) + lnotab = list(dis.findlinestarts(code)) + # Don't bother checking if the line info is sensible, because + # most of the line info we can get at comes from lnotab. + min_bytecode = min(t[0] for t in lnotab) + max_bytecode = max(t[0] for t in lnotab) + self.assertGreaterEqual(min_bytecode, 0) + self.assertLess(max_bytecode, len(code.co_code)) + # This could conceivably test more (and probably should, as there + # aren't very many tests of lnotab), if peepholer wasn't scheduled + # to be replaced anyway. + def test_unot(self): # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' def unot(x): @@ -48,6 +62,7 @@ def unot(x): self.assertNotInBytecode(unot, 'UNARY_NOT') self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') + self.check_lnotab(unot) def test_elim_inversion_of_is_or_in(self): for line, cmp_op in ( @@ -58,6 +73,7 @@ def test_elim_inversion_of_is_or_in(self): ): code = compile(line, '', 'single') self.assertInBytecode(code, 'COMPARE_OP', cmp_op) + self.check_lnotab(code) def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False @@ -75,6 +91,7 @@ def h(): for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + self.check_lnotab(func) def f(): 'Adding a docstring made this test fail in Py2.5.0' @@ -82,6 +99,7 @@ def f(): self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) + self.check_lnotab(f) def test_while_one(self): # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx @@ -93,6 +111,7 @@ def f(): self.assertNotInBytecode(f, elem) for elem in ('JUMP_ABSOLUTE',): self.assertInBytecode(f, elem) + self.check_lnotab(f) def test_pack_unpack(self): for line, elem in ( @@ -104,6 +123,7 @@ def test_pack_unpack(self): self.assertInBytecode(code, elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') self.assertNotInBytecode(code, 'UNPACK_TUPLE') + self.check_lnotab(code) def test_folding_of_tuples_of_constants(self): for line, elem in ( @@ -116,6 +136,7 @@ def test_folding_of_tuples_of_constants(self): code = compile(line,'','single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.check_lnotab(code) # Long tuples should be folded too. code = compile(repr(tuple(range(10000))),'','single') @@ -124,6 +145,7 @@ def test_folding_of_tuples_of_constants(self): load_consts = [instr for instr in dis.get_instructions(code) if instr.opname == 'LOAD_CONST'] self.assertEqual(len(load_consts), 2) + self.check_lnotab(code) # Bug 1053819: Tuple of constants misidentified when presented with: # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . @@ -141,6 +163,7 @@ def crater(): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ],) + self.check_lnotab(crater) def test_folding_of_lists_of_constants(self): for line, elem in ( @@ -153,6 +176,7 @@ def test_folding_of_lists_of_constants(self): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_LIST') + self.check_lnotab(code) def test_folding_of_sets_of_constants(self): for line, elem in ( @@ -166,6 +190,7 @@ def test_folding_of_sets_of_constants(self): code = compile(line, '', 'single') self.assertNotInBytecode(code, 'BUILD_SET') self.assertInBytecode(code, 'LOAD_CONST', elem) + self.check_lnotab(code) # Ensure that the resulting code actually works: def f(a): @@ -176,9 +201,11 @@ def g(a): self.assertTrue(f(3)) self.assertTrue(not f(4)) + self.check_lnotab(f) self.assertTrue(not g(3)) self.assertTrue(g(4)) + self.check_lnotab(g) def test_folding_of_binops_on_constants(self): @@ -203,41 +230,50 @@ def test_folding_of_binops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('BINARY_')) + self.check_lnotab(code) # Verify that unfoldables are skipped code = compile('a=2+"b"', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 2) self.assertInBytecode(code, 'LOAD_CONST', 'b') + self.check_lnotab(code) # Verify that large sequences do not result from folding code = compile('a="x"*10000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 10000) self.assertNotIn("x"*10000, code.co_consts) + self.check_lnotab(code) code = compile('a=1<<1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(1<<1000, code.co_consts) + self.check_lnotab(code) code = compile('a=2**1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(2**1000, code.co_consts) + self.check_lnotab(code) def test_binary_subscr_on_unicode(self): # valid code get optimized code = compile('"foo"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 'f') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) code = compile('"\u0061\uffff"[1]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\uffff') self.assertNotInBytecode(code,'BINARY_SUBSCR') + self.check_lnotab(code) # With PEP 393, non-BMP char get optimized code = compile('"\U00012345"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) # invalid code doesn't get optimized # out of range code = compile('"fuu"[10]', '', 'single') self.assertInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) def test_folding_of_unaryops_on_constants(self): for line, elem in ( @@ -252,13 +288,15 @@ def test_folding_of_unaryops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(code) # Check that -0.0 works after marshaling def negzero(): return -(1.0-1.0) - for instr in dis.get_instructions(code): + for instr in dis.get_instructions(negzero): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(negzero) # Verify that unfoldables are skipped for line, elem, opname in ( @@ -268,6 +306,7 @@ def negzero(): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertInBytecode(code, opname) + self.check_lnotab(code) def test_elim_extra_return(self): # RETURN LOAD_CONST None RETURN --> RETURN @@ -277,6 +316,7 @@ def f(x): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 1) + self.check_lnotab(f) def test_elim_jump_to_return(self): # JUMP_FORWARD to RETURN --> RETURN @@ -290,6 +330,7 @@ def f(cond, true_value, false_value): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 2) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump(self): # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump @@ -302,6 +343,7 @@ def f(): else: baz() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump2(self): # POP_JUMP_IF_FALSE to JUMP_ABSOLUTE --> POP_JUMP_IF_FALSE to non-jump @@ -312,6 +354,7 @@ def f(): or d): a = foo() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump3(self): # Intentionally use two-line expressions to test issue37213. @@ -320,18 +363,21 @@ def f(a, b, c): return ((a and b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_FALSE_OR_POP'), 2) # JUMP_IF_TRUE_OR_POP to JUMP_IF_TRUE_OR_POP --> JUMP_IF_TRUE_OR_POP to non-jump def f(a, b, c): return ((a or b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_TRUE_OR_POP'), 2) # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump def f(a, b, c): return ((a and b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_FALSE') @@ -340,6 +386,7 @@ def f(a, b, c): return ((a or b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_TRUE') @@ -360,6 +407,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 6) + self.check_lnotab(f) def test_elim_jump_after_return2(self): # Eliminate dead code: jumps immediately after returns can't be reached @@ -374,6 +422,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 2) + self.check_lnotab(f) def test_make_function_doesnt_bail(self): def f(): @@ -381,6 +430,7 @@ def g()->1+1: pass return g self.assertNotInBytecode(f, 'BINARY_ADD') + self.check_lnotab(f) def test_constant_folding(self): # Issue #11244: aggressive constant folding. @@ -401,17 +451,20 @@ def test_constant_folding(self): self.assertFalse(instr.opname.startswith('UNARY_')) self.assertFalse(instr.opname.startswith('BINARY_')) self.assertFalse(instr.opname.startswith('BUILD_')) + self.check_lnotab(code) def test_in_literal_list(self): def containtest(): return x in [a, b] self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + self.check_lnotab(containtest) def test_iterate_literal_list(self): def forloop(): for x in [a, b]: pass self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + self.check_lnotab(forloop) def test_condition_with_binop_with_bools(self): def f(): @@ -419,6 +472,7 @@ def f(): return 1 return 0 self.assertEqual(f(), 1) + self.check_lnotab(f) def test_if_with_if_expression(self): # Check bpo-37289 @@ -427,6 +481,19 @@ def f(x): return True return False self.assertTrue(f(True)) + self.check_lnotab(f) + + def test_trailing_nops(self): + # Check the lnotab of a function that even after trivial + # optimization has trailing nops, which the lnotab adjustment has to + # handle properly (bpo-38115). + def f(x): + while 1: + return 3 + while 1: + return 5 + return 6 + self.check_lnotab(f) class TestBuglets(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst new file mode 100644 index 000000000000..5119c0546e36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst @@ -0,0 +1 @@ +Fix a bug in dis.findlinestarts() where it would return invalid bytecode offsets. Document that a code object's co_lnotab can contain invalid bytecode offsets. \ No newline at end of file diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 3dab2b986616..71a297971828 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -3,7 +3,9 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes disguised as a Python bytes object. It is used to map bytecode offsets to source code line #s for tracebacks and to identify line number boundaries for -line tracing. +line tracing. Because of internals of the peephole optimizer, it's possible +for lnotab to contain bytecode offsets that are no longer valid (for example +if the optimizer removed the last line in a function). The array is conceptually a compressed list of (bytecode offset increment, line number increment)

1 0

bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448)
by Miss Islington (bot) Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/8f478b489ae11633d2609dff0ef21d0e1a… commit: 8f478b489ae11633d2609dff0ef21d0e1a857417 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-09-28T07:23:34-07:00 summary: bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) * bpo-38216: Allow bypassing input validation * bpo-36274: Also allow the URL encoding to be overridden. * bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL. * Call with skip_host to avoid tripping on the host checking in the URL. * Remove obsolete comment. * Make _prepare_path_encoding its own attr. This makes overriding just that simpler. Also, don't use the := operator to make backporting easier. * Add a news entry. * _prepare_path_encoding -> _encode_prepared_path() * Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path. (cherry picked from commit 7774d7831e8809795c64ce27f7df52674581d298) Co-authored-by: Jason R. Coombs <jaraco(a)jaraco.com> files: A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst M Lib/http/client.py M Lib/test/test_httplib.py diff --git a/Lib/http/client.py b/Lib/http/client.py index f61267e108a5..33a434733f8a 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1085,18 +1085,15 @@ def putrequest(self, method, url, skip_host=False, else: raise CannotSendRequest(self.__state) - # Save the method we use, we need it later in the response phase + # Save the method for use later in the response phase self._method = method - if not url: - url = '/' - # Prevent CVE-2019-9740. - if match := _contains_disallowed_url_pchar_re.search(url): - raise InvalidURL(f"URL can't contain control characters. {url!r} " - f"(found at least {match.group()!r})") + + url = url or '/' + self._validate_path(url) + request = '%s %s %s' % (method, url, self._http_vsn_str) - # Non-ASCII characters should have been eliminated earlier - self._output(request.encode('ascii')) + self._output(self._encode_request(request)) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance @@ -1174,6 +1171,18 @@ def putrequest(self, method, url, skip_host=False, # For HTTP/1.0, the server will assume "not chunked" pass + def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 1121224da443..e1aa41421feb 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1155,6 +1155,34 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_putrequest_override_validation(self): + """ + It should be possible to override the default validation + behavior in putrequest (bpo-38216). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_path(self, url): + pass + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/\x00') + + def test_putrequest_override_encoding(self): + """ + It should be possible to override the default encoding + to transmit bytes in another encoding even if invalid + (bpo-36274). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _encode_request(self, str_url): + return str_url.encode('utf-8') + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/☃') + + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1279,6 +1307,7 @@ def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst new file mode 100644 index 000000000000..ac8e2b042d92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst @@ -0,0 +1,4 @@ +Allow the rare code that wants to send invalid http requests from the +`http.client` library a way to do so. The fixes for bpo-30458 led to +breakage for some projects that were relying on this ability to test their +own behavior in the face of bad requests.

1 0

[3.7] bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) (GH-16461)
by Jason R. Coombs Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/80dd66ac278ecbabbf843526e3a56f5031… commit: 80dd66ac278ecbabbf843526e3a56f5031da9562 branch: 3.7 author: Jason R. Coombs <jaraco(a)jaraco.com> committer: GitHub <noreply(a)github.com> date: 2019-09-28T09:15:05-04:00 summary: [3.7] bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) (GH-16461) * bpo-38216: Allow bypassing input validation * bpo-36274: Also allow the URL encoding to be overridden. * bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL. * Call with skip_host to avoid tripping on the host checking in the URL. * Remove obsolete comment. * Make _prepare_path_encoding its own attr. This makes overriding just that simpler. Also, don't use the := operator to make backporting easier. * Add a news entry. * _prepare_path_encoding -> _encode_prepared_path() * Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path.. (cherry picked from commit 7774d7831e8809795c64ce27f7df52674581d298) Co-authored-by: Jason R. Coombs <jaraco(a)jaraco.com> files: A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst M Lib/http/client.py M Lib/test/test_httplib.py diff --git a/Lib/http/client.py b/Lib/http/client.py index dd23edcd597c..f7a183c77403 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1107,19 +1107,15 @@ def putrequest(self, method, url, skip_host=False, else: raise CannotSendRequest(self.__state) - # Save the method we use, we need it later in the response phase + # Save the method for use later in the response phase self._method = method - if not url: - url = '/' - # Prevent CVE-2019-9740. - match = _contains_disallowed_url_pchar_re.search(url) - if match: - raise InvalidURL(f"URL can't contain control characters. {url!r} " - f"(found at least {match.group()!r})") + + url = url or '/' + self._validate_path(url) + request = '%s %s %s' % (method, url, self._http_vsn_str) - # Non-ASCII characters should have been eliminated earlier - self._output(request.encode('ascii')) + self._output(self._encode_request(request)) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance @@ -1197,6 +1193,18 @@ def putrequest(self, method, url, skip_host=False, # For HTTP/1.0, the server will assume "not chunked" pass + def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index c4246671586f..9facb2e36017 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1157,6 +1157,34 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_putrequest_override_validation(self): + """ + It should be possible to override the default validation + behavior in putrequest (bpo-38216). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_path(self, url): + pass + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/\x00') + + def test_putrequest_override_encoding(self): + """ + It should be possible to override the default encoding + to transmit bytes in another encoding even if invalid + (bpo-36274). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _encode_request(self, str_url): + return str_url.encode('utf-8') + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/☃') + + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1281,6 +1309,7 @@ def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst new file mode 100644 index 000000000000..ac8e2b042d92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst @@ -0,0 +1,4 @@ +Allow the rare code that wants to send invalid http requests from the +`http.client` library a way to do so. The fixes for bpo-30458 led to +breakage for some projects that were relying on this ability to test their +own behavior in the face of bad requests.

1 0

bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448)
by Jason R. Coombs Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/7774d7831e8809795c64ce27f7df526745… commit: 7774d7831e8809795c64ce27f7df52674581d298 branch: master author: Jason R. Coombs <jaraco(a)jaraco.com> committer: GitHub <noreply(a)github.com> date: 2019-09-28T08:32:01-04:00 summary: bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) * bpo-38216: Allow bypassing input validation * bpo-36274: Also allow the URL encoding to be overridden. * bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL. * Call with skip_host to avoid tripping on the host checking in the URL. * Remove obsolete comment. * Make _prepare_path_encoding its own attr. This makes overriding just that simpler. Also, don't use the := operator to make backporting easier. * Add a news entry. * _prepare_path_encoding -> _encode_prepared_path() * Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path. files: A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst M Lib/http/client.py M Lib/test/test_httplib.py diff --git a/Lib/http/client.py b/Lib/http/client.py index f61267e108a5..33a434733f8a 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1085,18 +1085,15 @@ def putrequest(self, method, url, skip_host=False, else: raise CannotSendRequest(self.__state) - # Save the method we use, we need it later in the response phase + # Save the method for use later in the response phase self._method = method - if not url: - url = '/' - # Prevent CVE-2019-9740. - if match := _contains_disallowed_url_pchar_re.search(url): - raise InvalidURL(f"URL can't contain control characters. {url!r} " - f"(found at least {match.group()!r})") + + url = url or '/' + self._validate_path(url) + request = '%s %s %s' % (method, url, self._http_vsn_str) - # Non-ASCII characters should have been eliminated earlier - self._output(request.encode('ascii')) + self._output(self._encode_request(request)) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance @@ -1174,6 +1171,18 @@ def putrequest(self, method, url, skip_host=False, # For HTTP/1.0, the server will assume "not chunked" pass + def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 1121224da443..e1aa41421feb 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1155,6 +1155,34 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_putrequest_override_validation(self): + """ + It should be possible to override the default validation + behavior in putrequest (bpo-38216). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_path(self, url): + pass + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/\x00') + + def test_putrequest_override_encoding(self): + """ + It should be possible to override the default encoding + to transmit bytes in another encoding even if invalid + (bpo-36274). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _encode_request(self, str_url): + return str_url.encode('utf-8') + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/☃') + + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1279,6 +1307,7 @@ def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst new file mode 100644 index 000000000000..ac8e2b042d92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst @@ -0,0 +1,4 @@ +Allow the rare code that wants to send invalid http requests from the +`http.client` library a way to do so. The fixes for bpo-30458 led to +breakage for some projects that were relying on this ability to test their +own behavior in the face of bad requests.

1 0

bpo-38243, xmlrpc.server: Escape the server_title (GH-16373) (GH-16441)
by Ned Deily Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/1698cacfb924d1df452e78d11a4bf81ae7… commit: 1698cacfb924d1df452e78d11a4bf81ae7777389 branch: 3.6 author: Victor Stinner <vstinner(a)redhat.com> committer: Ned Deily <nad(a)python.org> date: 2019-09-28T03:33:00-04:00 summary: bpo-38243, xmlrpc.server: Escape the server_title (GH-16373) (GH-16441) Escape the server title of xmlrpc.server.DocXMLRPCServer when rendering the document page as HTML. (cherry picked from commit e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa) files: A Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst M Lib/test/test_docxmlrpc.py M Lib/xmlrpc/server.py diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 00903337c07c..d2adb21af0fb 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -1,5 +1,6 @@ from xmlrpc.server import DocXMLRPCServer import http.client +import re import sys from test import support threading = support.import_module('threading') @@ -193,6 +194,21 @@ def test_annotations(self): b'method_annotation</strong></a>(x: bytes)</dt></dl>'), response.read()) + def test_server_title_escape(self): + # bpo-38243: Ensure that the server title and documentation + # are escaped for HTML. + self.serv.set_server_title('test_title<script>') + self.serv.set_server_documentation('test_documentation<script>') + self.assertEqual('test_title<script>', self.serv.server_title) + self.assertEqual('test_documentation<script>', + self.serv.server_documentation) + + generated = self.serv.generate_html_documentation() + title = re.search(r'<title>(.+?)</title>', generated).group() + documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group() + self.assertEqual('<title>Python: test_title<script></title>', title) + self.assertEqual('<p><tt>test_documentation<script></tt></p>', documentation) + if __name__ == '__main__': unittest.main() diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 3e0dca027f06..efe593748968 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -106,6 +106,7 @@ def export_add(self, x, y): from xmlrpc.client import Fault, dumps, loads, gzip_encode, gzip_decode from http.server import BaseHTTPRequestHandler +import html import http.server import socketserver import sys @@ -904,7 +905,7 @@ def generate_html_documentation(self): methods ) - return documenter.page(self.server_title, documentation) + return documenter.page(html.escape(self.server_title), documentation) class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): """XML-RPC and documentation request handler class. diff --git a/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst new file mode 100644 index 000000000000..98d7be129573 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst @@ -0,0 +1,3 @@ +Escape the server title of :class:`xmlrpc.server.DocXMLRPCServer` +when rendering the document page as HTML. +(Contributed by Dong-hee Na in :issue:`38243`.)

1 0

(no subject)
by Jesús Cea Sept. 28, 2019

Sept. 28, 2019

To: python-checkins(a)python.org Subject: [2.7] bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (GH-16446). (#16454) Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 https://github.com/python/cpython/commit/598f676880662fb453ff98fda42b7b7068… be32 commit: 598f676880662fb453ff98fda42b7b7068e5be32 branch: 2.7 author: Jes=C3=BAs Cea <jcea(a)jcea.es> committer: GitHub <noreply(a)github.com> date: 2019-09-28T05:09:24+02:00 summary: [2.7] bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (GH= -16446). (#16454) (cherry picked from commit 52d1b86bde2b772a76919c76991c326384954bf1) Co-authored-by: Jes=C3=BAs Cea <jcea(a)jcea.es> files: A Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst M configure M configure.ac diff --git a/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst = b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst new file mode 100644 index 000000000000..59c9a76385e2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst @@ -0,0 +1,2 @@ +In Solaris family, we must be sure to use ``-D_REENTRANT``. +Patch by Jes=C3=BAs Cea Avi=C3=B3n. diff --git a/configure b/configure index 67300fe2b6e1..63d675312da9 100755 --- a/configure +++ b/configure @@ -9602,6 +9602,9 @@ then =20 posix_threads=3Dyes THREADOBJ=3D"Python/thread.o" + if test "$ac_sys_system" =3D "SunOS"; then + CFLAGS=3D"$CFLAGS -D_REENTRANT" + fi elif test "$ac_cv_kpthread" =3D "yes" then CC=3D"$CC -Kpthread" diff --git a/configure.ac b/configure.ac index 36df3d02a2d8..efe6922b5de2 100644 --- a/configure.ac +++ b/configure.ac @@ -2625,6 +2625,9 @@ then AC_DEFINE(_REENTRANT) posix_threads=3Dyes THREADOBJ=3D"Python/thread.o" =20 + if test "$ac_sys_system" =3D "SunOS"; then + CFLAGS=3D"$CFLAGS -D_REENTRANT" + fi elif test "$ac_cv_kpthread" =3D "yes" then CC=3D"$CC -Kpthread"

1 0

bpo-38304: Add PyConfig.struct_size (GH-16451) (GH-16453)
by Victor Stinner Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/6e128382b3e7f83d91d5e73d83a051ff5f… commit: 6e128382b3e7f83d91d5e73d83a051ff5fb6469c branch: 3.8 author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-09-28T04:50:43+02:00 summary: bpo-38304: Add PyConfig.struct_size (GH-16451) (GH-16453) Add a new struct_size field to PyPreConfig and PyConfig structures to allow to modify these structures in the future without breaking the backward compatibility. * Replace private _config_version field with public struct_size field in PyPreConfig and PyConfig. * Public PyPreConfig_InitIsolatedConfig() and PyPreConfig_InitPythonConfig() return type becomes PyStatus, instead of void. * Internal _PyConfig_InitCompatConfig(), _PyPreConfig_InitCompatConfig(), _PyPreConfig_InitFromConfig(), _PyPreConfig_InitFromPreConfig() return type becomes PyStatus, instead of void. * Remove _Py_CONFIG_VERSION * Update the Initialization Configuration documentation. (cherry picked from commit 441b10cf2855955c86565f8d59e72c2efc0f0a57) files: A Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst M Doc/c-api/init_config.rst M Include/cpython/initconfig.h M Include/internal/pycore_initconfig.h M Modules/main.c M Programs/_freeze_importlib.c M Programs/_testembed.c M Python/frozenmain.c M Python/initconfig.c M Python/pathconfig.c M Python/preconfig.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0c3c725c841a..58e417441d13 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -194,18 +194,25 @@ PyPreConfig * Configure the LC_CTYPE locale * Set the UTF-8 mode + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyPreConfig)``. + Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Python Configuration <init-python-config>`. - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Isolated Configuration <init-isolated-conf>`. + The caller of these functions is responsible to handle exceptions (error or + exit) using :c:func:`PyStatus_Exception` and + :c:func:`Py_ExitStatusException`. + Structure fields: .. c:member:: int allocator @@ -267,6 +274,13 @@ PyPreConfig same way the regular Python parses command line arguments: see :ref:`Command Line Arguments <using-on-cmdline>`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyPreConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int use_environment See :c:member:`PyConfig.use_environment`. @@ -316,12 +330,18 @@ the preinitialization. Example using the preinitialization to enable the UTF-8 Mode:: + PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.utf8_mode = 1; - PyStatus status = Py_PreInitialize(&preconfig); + status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); } @@ -340,6 +360,9 @@ PyConfig Structure containing most parameters to configure Python. + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyConfig)``. + Structure methods: .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config) @@ -656,6 +679,13 @@ PyConfig Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int tracemalloc If non-zero, call :func:`tracemalloc.start` at startup. @@ -718,6 +748,7 @@ Example setting the program name:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -750,6 +781,7 @@ configuration, and then override some parameters:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -835,8 +867,9 @@ Example of customized Python always running in isolated mode:: int main(int argc, char **argv) { - PyConfig config; PyStatus status; + PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1028,6 +1061,7 @@ phases:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 047f511f3104..d5effb8397fc 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -45,8 +45,10 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list, /* --- PyPreConfig ----------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyPreConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ /* Parse Py_PreInitializeFromBytesArgs() arguments? @@ -122,15 +124,17 @@ typedef struct { int allocator; } PyPreConfig; -PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config); -PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); /* --- PyConfig ---------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ int isolated; /* Isolated mode? see PyPreConfig.isolated */ @@ -403,7 +407,6 @@ typedef struct { /* If equal to 0, stop Python initialization before the "main" phase */ int _init_main; - } PyConfig; PyAPI_FUNC(PyStatus) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index a7c1994eca16..dcdb61695cab 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -40,6 +40,8 @@ extern "C" { (err._type == _PyStatus_TYPE_EXIT) #define _PyStatus_EXCEPTION(err) \ (err._type != _PyStatus_TYPE_OK) +#define _PyStatus_UPDATE_FUNC(err) \ + do { err.func = _PyStatus_GET_FUNC(); } while (0) /* --- PyWideStringList ------------------------------------------------ */ @@ -118,11 +120,11 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline, /* --- PyPreConfig ----------------------------------------------- */ -PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); -extern void _PyPreConfig_InitFromConfig( +PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); +extern PyStatus _PyPreConfig_InitFromConfig( PyPreConfig *preconfig, const PyConfig *config); -extern void _PyPreConfig_InitFromPreConfig( +extern PyStatus _PyPreConfig_InitFromPreConfig( PyPreConfig *preconfig, const PyPreConfig *config2); extern PyObject* _PyPreConfig_AsDict(const PyPreConfig *preconfig); @@ -135,8 +137,6 @@ extern PyStatus _PyPreConfig_Write(const PyPreConfig *preconfig); /* --- PyConfig ---------------------------------------------- */ -#define _Py_CONFIG_VERSION 1 - typedef enum { /* Py_Initialize() API: backward compatibility with Python 3.6 and 3.7 */ _PyConfig_INIT_COMPAT = 1, @@ -144,7 +144,7 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; -PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); +PyAPI_FUNC(PyStatus) _PyConfig_InitCompatConfig(PyConfig *config); extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); diff --git a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst new file mode 100644 index 000000000000..b797933d05d6 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst @@ -0,0 +1,3 @@ +Add a new ``struct_size`` field to :c:type:`PyPreConfig` and :c:type:`PyConfig` +structures to allow to modify these structures in the future without breaking +the backward compatibility. diff --git a/Modules/main.c b/Modules/main.c index b8a1c9b79ce3..13936add4f3e 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,13 +53,20 @@ pymain_init(const _PyArgv *args) #endif PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _Py_PreInitializeFromPyArgv(&preconfig, args); if (_PyStatus_EXCEPTION(status)) { return status; } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (_PyStatus_EXCEPTION(status)) { goto done; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 74735f279c58..7c494c2786c0 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -78,6 +78,7 @@ main(int argc, char *argv[]) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 83c266b885af..a37594524f3d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -385,7 +385,12 @@ static int check_init_compat_config(int preinit) if (preinit) { PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -394,7 +399,13 @@ static int check_init_compat_config(int preinit) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + config_set_program_name(&config); init_from_config_clear(&config); @@ -470,7 +481,12 @@ static int test_init_from_config(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } putenv("PYTHONMALLOC=malloc_debug"); preconfig.allocator = PYMEM_ALLOCATOR_MALLOC; @@ -485,7 +501,12 @@ static int test_init_from_config(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config.install_signal_handlers = 0; /* FIXME: test use_environment */ @@ -617,6 +638,8 @@ static int check_init_parse_argv(int parse_argv) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -702,6 +725,8 @@ static int test_init_python_env(void) set_all_env_vars(); PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -755,6 +780,8 @@ static int test_init_isolated_flag(void) /* Test PyConfig.isolated=1 */ PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -779,7 +806,13 @@ static int test_preinit_isolated1(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 1; status = Py_PreInitialize(&preconfig); @@ -788,7 +821,12 @@ static int test_preinit_isolated1(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config_set_program_name(&config); set_all_env_vars(); init_from_config_clear(&config); @@ -805,7 +843,13 @@ static int test_preinit_isolated2(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 0; status = Py_PreInitialize(&preconfig); @@ -815,7 +859,11 @@ static int test_preinit_isolated2(void) /* Test PyConfig.isolated=1 */ PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } Py_IsolatedFlag = 0; config.isolated = 1; @@ -835,7 +883,12 @@ static int test_preinit_dont_parse_argv(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.isolated = 0; @@ -852,6 +905,7 @@ static int test_preinit_dont_parse_argv(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { @@ -877,6 +931,7 @@ static int test_preinit_parse_argv(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -934,7 +989,12 @@ static int check_preinit_isolated_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -947,6 +1007,8 @@ static int check_preinit_isolated_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); @@ -996,7 +1058,12 @@ static int check_init_python_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -1005,6 +1072,8 @@ static int check_init_python_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1035,7 +1104,13 @@ static int test_init_dont_configure_locale(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.configure_locale = 0; preconfig.coerce_c_locale = 1; preconfig.coerce_c_locale_warn = 1; @@ -1046,6 +1121,8 @@ static int test_init_dont_configure_locale(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1063,6 +1140,8 @@ static int test_init_dev_mode(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1286,6 +1365,8 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1334,6 +1415,8 @@ static int test_init_read_set(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1382,6 +1465,8 @@ static int test_init_sys_add(void) PySys_AddWarnOption(L"ignore:::sysadd_warnoption"); PyConfig config; + config.struct_size = sizeof(PyConfig); + PyStatus status; status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1450,7 +1535,12 @@ static int test_init_setpath_config(void) { PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } /* Explicitly preinitializes with Python preconfiguration to avoid Py_SetPath() implicit preinitialization with compat preconfiguration. */ @@ -1474,6 +1564,7 @@ static int test_init_setpath_config(void) putenv("TESTPATH="); PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1531,6 +1622,8 @@ static int test_init_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1546,6 +1639,7 @@ static int test_init_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1577,6 +1671,7 @@ static int test_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Python/frozenmain.c b/Python/frozenmain.c index c56938ab4899..76309e9e5da2 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -40,6 +40,7 @@ Py_FrozenMain(int argc, char **argv) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); diff --git a/Python/initconfig.c b/Python/initconfig.c index a527271452ea..dbc9de55a582 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -528,6 +528,18 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv) ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY()) + +static PyStatus +config_check_struct_size(const PyConfig *config) +{ + if (config->struct_size != sizeof(PyConfig)) { + return _PyStatus_ERR("unsupported PyConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + /* Free memory allocated in config, but don't clear all attributes */ void PyConfig_Clear(PyConfig *config) @@ -568,12 +580,19 @@ PyConfig_Clear(PyConfig *config) } -void +PyStatus _PyConfig_InitCompatConfig(PyConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->isolated = -1; config->use_environment = -1; @@ -603,13 +622,17 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + return _PyStatus_OK(); } -static void +static PyStatus config_init_defaults(PyConfig *config) { - _PyConfig_InitCompatConfig(config); + PyStatus status = _PyConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } config->isolated = 0; config->use_environment = 1; @@ -628,13 +651,18 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif + return _PyStatus_OK(); } PyStatus PyConfig_InitPythonConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->configure_c_stdio = 1; @@ -647,7 +675,11 @@ PyConfig_InitPythonConfig(PyConfig *config) PyStatus PyConfig_InitIsolatedConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->isolated = 1; @@ -742,6 +774,19 @@ PyStatus _PyConfig_Copy(PyConfig *config, const PyConfig *config2) { PyStatus status; + + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + + status = config_check_struct_size(config2); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + PyConfig_Clear(config); #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR @@ -2199,7 +2244,12 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline) } PyPreConfig preconfig; - _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } _PyPreConfig_GetConfig(&preconfig, config); @@ -2338,6 +2388,12 @@ PyConfig_Read(PyConfig *config) PyStatus status; PyWideStringList orig_argv = PyWideStringList_INIT; + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + status = _Py_PreInitializeFromConfig(config, NULL); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index d5b8b1acff8a..41ef7d28d066 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -434,7 +434,12 @@ pathconfig_global_read(_PyPathConfig *pathconfig) { PyStatus status; PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } /* Call _PyConfig_InitPathConfig() */ status = PyConfig_Read(&config); diff --git a/Python/preconfig.c b/Python/preconfig.c index 8be6533eace0..d18b01dc4586 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -269,12 +269,30 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig) /* --- PyPreConfig ----------------------------------------------- */ -void +static PyStatus +preconfig_check_struct_size(PyPreConfig *config) +{ + if (config->struct_size != sizeof(PyPreConfig)) { + return _PyStatus_ERR("unsupported PyPreConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + +PyStatus _PyPreConfig_InitCompatConfig(PyPreConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->parse_argv = 0; config->isolated = -1; @@ -295,13 +313,18 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = -1; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->isolated = 0; @@ -316,13 +339,18 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->configure_locale = 0; @@ -333,41 +361,55 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromPreConfig(PyPreConfig *config, const PyPreConfig *config2) { - PyPreConfig_InitPythonConfig(config); + PyStatus status = PyPreConfig_InitPythonConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + preconfig_copy(config, config2); + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config) { + PyStatus status; _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init; switch (config_init) { case _PyConfig_INIT_PYTHON: - PyPreConfig_InitPythonConfig(preconfig); + status = PyPreConfig_InitPythonConfig(preconfig); break; case _PyConfig_INIT_ISOLATED: - PyPreConfig_InitIsolatedConfig(preconfig); + status = PyPreConfig_InitIsolatedConfig(preconfig); break; case _PyConfig_INIT_COMPAT: default: - _PyPreConfig_InitCompatConfig(preconfig); + status = _PyPreConfig_InitCompatConfig(preconfig); } + + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyPreConfig_GetConfig(preconfig, config); + return _PyStatus_OK(); } static void preconfig_copy(PyPreConfig *config, const PyPreConfig *config2) { - assert(config2->_config_version == _Py_CONFIG_VERSION); + assert(config->struct_size == sizeof(PyPreConfig)); + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR COPY_ATTR(_config_init); @@ -787,6 +829,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) return status; } + status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + preconfig_get_global_vars(config); /* Copy LC_CTYPE locale, since it's modified later */ @@ -801,7 +849,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) /* Save the config to be able to restore it if encodings change */ PyPreConfig save_config; - _PyPreConfig_InitFromPreConfig(&save_config, config); + save_config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&save_config, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } /* Set LC_CTYPE to the user preferred locale */ if (config->configure_locale) { @@ -923,7 +976,12 @@ PyStatus _PyPreConfig_Write(const PyPreConfig *src_config) { PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (_PyRuntime.core_initialized) { /* bpo-34008: Calling this functions after Py_Initialize() ignores diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 42a062e10865..c154d060ad8d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -731,7 +731,12 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) runtime->preinitializing = 1; PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } status = _PyPreConfig_Read(&config, args); if (_PyStatus_EXCEPTION(status)) { @@ -790,7 +795,12 @@ _Py_PreInitializeFromConfig(const PyConfig *config, } PyPreConfig preconfig; - _PyPreConfig_InitFromConfig(&preconfig, config); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromConfig(&preconfig, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (!config->parse_argv) { return Py_PreInitialize(&preconfig); @@ -838,7 +848,12 @@ pyinit_core(_PyRuntimeState *runtime, } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } status = _PyConfig_Copy(&config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -1061,7 +1076,13 @@ Py_InitializeEx(int install_sigs) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } + config.install_signal_handlers = install_sigs; status = Py_InitializeFromConfig(&config); diff --git a/Python/pystate.c b/Python/pystate.c index f2924a8ff263..f0662fa70657 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -60,7 +60,12 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); - PyPreConfig_InitPythonConfig(&runtime->preconfig); + + runtime->preconfig.struct_size = sizeof(PyPreConfig); + PyStatus status = PyPreConfig_InitPythonConfig(&runtime->preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } runtime->gilstate.check_enabled = 1; @@ -205,6 +210,7 @@ PyInterpreterState_New(void) interp->id_refcount = -1; interp->check_interval = 100; + interp->config.struct_size = sizeof(PyConfig); PyStatus status = PyConfig_InitPythonConfig(&interp->config); if (_PyStatus_EXCEPTION(status)) { /* Don't report status to caller: PyConfig_InitPythonConfig()

1 0

bpo-38304: Add PyConfig.struct_size (GH-16451)
by Victor Stinner Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/441b10cf2855955c86565f8d59e72c2efc… commit: 441b10cf2855955c86565f8d59e72c2efc0f0a57 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-09-28T04:28:35+02:00 summary: bpo-38304: Add PyConfig.struct_size (GH-16451) Add a new struct_size field to PyPreConfig and PyConfig structures to allow to modify these structures in the future without breaking the backward compatibility. * Replace private _config_version field with public struct_size field in PyPreConfig and PyConfig. * Public PyPreConfig_InitIsolatedConfig() and PyPreConfig_InitPythonConfig() return type becomes PyStatus, instead of void. * Internal _PyConfig_InitCompatConfig(), _PyPreConfig_InitCompatConfig(), _PyPreConfig_InitFromConfig(), _PyPreConfig_InitFromPreConfig() return type becomes PyStatus, instead of void. * Remove _Py_CONFIG_VERSION * Update the Initialization Configuration documentation. files: A Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst M Doc/c-api/init_config.rst M Include/cpython/initconfig.h M Include/internal/pycore_initconfig.h M Modules/main.c M Programs/_freeze_importlib.c M Programs/_testembed.c M Python/frozenmain.c M Python/initconfig.c M Python/pathconfig.c M Python/preconfig.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0c3c725c841a..58e417441d13 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -194,18 +194,25 @@ PyPreConfig * Configure the LC_CTYPE locale * Set the UTF-8 mode + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyPreConfig)``. + Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Python Configuration <init-python-config>`. - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Isolated Configuration <init-isolated-conf>`. + The caller of these functions is responsible to handle exceptions (error or + exit) using :c:func:`PyStatus_Exception` and + :c:func:`Py_ExitStatusException`. + Structure fields: .. c:member:: int allocator @@ -267,6 +274,13 @@ PyPreConfig same way the regular Python parses command line arguments: see :ref:`Command Line Arguments <using-on-cmdline>`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyPreConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int use_environment See :c:member:`PyConfig.use_environment`. @@ -316,12 +330,18 @@ the preinitialization. Example using the preinitialization to enable the UTF-8 Mode:: + PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.utf8_mode = 1; - PyStatus status = Py_PreInitialize(&preconfig); + status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); } @@ -340,6 +360,9 @@ PyConfig Structure containing most parameters to configure Python. + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyConfig)``. + Structure methods: .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config) @@ -656,6 +679,13 @@ PyConfig Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int tracemalloc If non-zero, call :func:`tracemalloc.start` at startup. @@ -718,6 +748,7 @@ Example setting the program name:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -750,6 +781,7 @@ configuration, and then override some parameters:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -835,8 +867,9 @@ Example of customized Python always running in isolated mode:: int main(int argc, char **argv) { - PyConfig config; PyStatus status; + PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1028,6 +1061,7 @@ phases:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 047f511f3104..d5effb8397fc 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -45,8 +45,10 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list, /* --- PyPreConfig ----------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyPreConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ /* Parse Py_PreInitializeFromBytesArgs() arguments? @@ -122,15 +124,17 @@ typedef struct { int allocator; } PyPreConfig; -PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config); -PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); /* --- PyConfig ---------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ int isolated; /* Isolated mode? see PyPreConfig.isolated */ @@ -403,7 +407,6 @@ typedef struct { /* If equal to 0, stop Python initialization before the "main" phase */ int _init_main; - } PyConfig; PyAPI_FUNC(PyStatus) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index a7c1994eca16..dcdb61695cab 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -40,6 +40,8 @@ extern "C" { (err._type == _PyStatus_TYPE_EXIT) #define _PyStatus_EXCEPTION(err) \ (err._type != _PyStatus_TYPE_OK) +#define _PyStatus_UPDATE_FUNC(err) \ + do { err.func = _PyStatus_GET_FUNC(); } while (0) /* --- PyWideStringList ------------------------------------------------ */ @@ -118,11 +120,11 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline, /* --- PyPreConfig ----------------------------------------------- */ -PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); -extern void _PyPreConfig_InitFromConfig( +PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); +extern PyStatus _PyPreConfig_InitFromConfig( PyPreConfig *preconfig, const PyConfig *config); -extern void _PyPreConfig_InitFromPreConfig( +extern PyStatus _PyPreConfig_InitFromPreConfig( PyPreConfig *preconfig, const PyPreConfig *config2); extern PyObject* _PyPreConfig_AsDict(const PyPreConfig *preconfig); @@ -135,8 +137,6 @@ extern PyStatus _PyPreConfig_Write(const PyPreConfig *preconfig); /* --- PyConfig ---------------------------------------------- */ -#define _Py_CONFIG_VERSION 1 - typedef enum { /* Py_Initialize() API: backward compatibility with Python 3.6 and 3.7 */ _PyConfig_INIT_COMPAT = 1, @@ -144,7 +144,7 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; -PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); +PyAPI_FUNC(PyStatus) _PyConfig_InitCompatConfig(PyConfig *config); extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); diff --git a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst new file mode 100644 index 000000000000..b797933d05d6 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst @@ -0,0 +1,3 @@ +Add a new ``struct_size`` field to :c:type:`PyPreConfig` and :c:type:`PyConfig` +structures to allow to modify these structures in the future without breaking +the backward compatibility. diff --git a/Modules/main.c b/Modules/main.c index fdc7e0dabf17..ef6c66a8dbe5 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,13 +53,20 @@ pymain_init(const _PyArgv *args) #endif PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _Py_PreInitializeFromPyArgv(&preconfig, args); if (_PyStatus_EXCEPTION(status)) { return status; } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (_PyStatus_EXCEPTION(status)) { goto done; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 74735f279c58..7c494c2786c0 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -78,6 +78,7 @@ main(int argc, char *argv[]) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 83c266b885af..a37594524f3d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -385,7 +385,12 @@ static int check_init_compat_config(int preinit) if (preinit) { PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -394,7 +399,13 @@ static int check_init_compat_config(int preinit) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + config_set_program_name(&config); init_from_config_clear(&config); @@ -470,7 +481,12 @@ static int test_init_from_config(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } putenv("PYTHONMALLOC=malloc_debug"); preconfig.allocator = PYMEM_ALLOCATOR_MALLOC; @@ -485,7 +501,12 @@ static int test_init_from_config(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config.install_signal_handlers = 0; /* FIXME: test use_environment */ @@ -617,6 +638,8 @@ static int check_init_parse_argv(int parse_argv) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -702,6 +725,8 @@ static int test_init_python_env(void) set_all_env_vars(); PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -755,6 +780,8 @@ static int test_init_isolated_flag(void) /* Test PyConfig.isolated=1 */ PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -779,7 +806,13 @@ static int test_preinit_isolated1(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 1; status = Py_PreInitialize(&preconfig); @@ -788,7 +821,12 @@ static int test_preinit_isolated1(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config_set_program_name(&config); set_all_env_vars(); init_from_config_clear(&config); @@ -805,7 +843,13 @@ static int test_preinit_isolated2(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 0; status = Py_PreInitialize(&preconfig); @@ -815,7 +859,11 @@ static int test_preinit_isolated2(void) /* Test PyConfig.isolated=1 */ PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } Py_IsolatedFlag = 0; config.isolated = 1; @@ -835,7 +883,12 @@ static int test_preinit_dont_parse_argv(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.isolated = 0; @@ -852,6 +905,7 @@ static int test_preinit_dont_parse_argv(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { @@ -877,6 +931,7 @@ static int test_preinit_parse_argv(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -934,7 +989,12 @@ static int check_preinit_isolated_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -947,6 +1007,8 @@ static int check_preinit_isolated_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); @@ -996,7 +1058,12 @@ static int check_init_python_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -1005,6 +1072,8 @@ static int check_init_python_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1035,7 +1104,13 @@ static int test_init_dont_configure_locale(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.configure_locale = 0; preconfig.coerce_c_locale = 1; preconfig.coerce_c_locale_warn = 1; @@ -1046,6 +1121,8 @@ static int test_init_dont_configure_locale(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1063,6 +1140,8 @@ static int test_init_dev_mode(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1286,6 +1365,8 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1334,6 +1415,8 @@ static int test_init_read_set(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1382,6 +1465,8 @@ static int test_init_sys_add(void) PySys_AddWarnOption(L"ignore:::sysadd_warnoption"); PyConfig config; + config.struct_size = sizeof(PyConfig); + PyStatus status; status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1450,7 +1535,12 @@ static int test_init_setpath_config(void) { PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } /* Explicitly preinitializes with Python preconfiguration to avoid Py_SetPath() implicit preinitialization with compat preconfiguration. */ @@ -1474,6 +1564,7 @@ static int test_init_setpath_config(void) putenv("TESTPATH="); PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1531,6 +1622,8 @@ static int test_init_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1546,6 +1639,7 @@ static int test_init_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1577,6 +1671,7 @@ static int test_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Python/frozenmain.c b/Python/frozenmain.c index c56938ab4899..76309e9e5da2 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -40,6 +40,7 @@ Py_FrozenMain(int argc, char **argv) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); diff --git a/Python/initconfig.c b/Python/initconfig.c index 9f04e3d183ab..62c868d5cbc9 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -528,6 +528,18 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv) ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY()) + +static PyStatus +config_check_struct_size(const PyConfig *config) +{ + if (config->struct_size != sizeof(PyConfig)) { + return _PyStatus_ERR("unsupported PyConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + /* Free memory allocated in config, but don't clear all attributes */ void PyConfig_Clear(PyConfig *config) @@ -568,12 +580,19 @@ PyConfig_Clear(PyConfig *config) } -void +PyStatus _PyConfig_InitCompatConfig(PyConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->isolated = -1; config->use_environment = -1; @@ -603,13 +622,17 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + return _PyStatus_OK(); } -static void +static PyStatus config_init_defaults(PyConfig *config) { - _PyConfig_InitCompatConfig(config); + PyStatus status = _PyConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } config->isolated = 0; config->use_environment = 1; @@ -628,13 +651,18 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif + return _PyStatus_OK(); } PyStatus PyConfig_InitPythonConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->configure_c_stdio = 1; @@ -647,7 +675,11 @@ PyConfig_InitPythonConfig(PyConfig *config) PyStatus PyConfig_InitIsolatedConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->isolated = 1; @@ -742,6 +774,19 @@ PyStatus _PyConfig_Copy(PyConfig *config, const PyConfig *config2) { PyStatus status; + + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + + status = config_check_struct_size(config2); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + PyConfig_Clear(config); #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR @@ -2204,7 +2249,12 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline) } PyPreConfig preconfig; - _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } _PyPreConfig_GetConfig(&preconfig, config); @@ -2385,6 +2435,12 @@ PyConfig_Read(PyConfig *config) PyStatus status; PyWideStringList orig_argv = PyWideStringList_INIT; + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + status = _Py_PreInitializeFromConfig(config, NULL); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 812614532800..6886ab7c42d6 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -434,7 +434,12 @@ pathconfig_global_read(_PyPathConfig *pathconfig) { PyStatus status; PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } /* Call _PyConfig_InitPathConfig() */ status = PyConfig_Read(&config); diff --git a/Python/preconfig.c b/Python/preconfig.c index 8be6533eace0..d18b01dc4586 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -269,12 +269,30 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig) /* --- PyPreConfig ----------------------------------------------- */ -void +static PyStatus +preconfig_check_struct_size(PyPreConfig *config) +{ + if (config->struct_size != sizeof(PyPreConfig)) { + return _PyStatus_ERR("unsupported PyPreConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + +PyStatus _PyPreConfig_InitCompatConfig(PyPreConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->parse_argv = 0; config->isolated = -1; @@ -295,13 +313,18 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = -1; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->isolated = 0; @@ -316,13 +339,18 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->configure_locale = 0; @@ -333,41 +361,55 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromPreConfig(PyPreConfig *config, const PyPreConfig *config2) { - PyPreConfig_InitPythonConfig(config); + PyStatus status = PyPreConfig_InitPythonConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + preconfig_copy(config, config2); + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config) { + PyStatus status; _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init; switch (config_init) { case _PyConfig_INIT_PYTHON: - PyPreConfig_InitPythonConfig(preconfig); + status = PyPreConfig_InitPythonConfig(preconfig); break; case _PyConfig_INIT_ISOLATED: - PyPreConfig_InitIsolatedConfig(preconfig); + status = PyPreConfig_InitIsolatedConfig(preconfig); break; case _PyConfig_INIT_COMPAT: default: - _PyPreConfig_InitCompatConfig(preconfig); + status = _PyPreConfig_InitCompatConfig(preconfig); } + + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyPreConfig_GetConfig(preconfig, config); + return _PyStatus_OK(); } static void preconfig_copy(PyPreConfig *config, const PyPreConfig *config2) { - assert(config2->_config_version == _Py_CONFIG_VERSION); + assert(config->struct_size == sizeof(PyPreConfig)); + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR COPY_ATTR(_config_init); @@ -787,6 +829,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) return status; } + status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + preconfig_get_global_vars(config); /* Copy LC_CTYPE locale, since it's modified later */ @@ -801,7 +849,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) /* Save the config to be able to restore it if encodings change */ PyPreConfig save_config; - _PyPreConfig_InitFromPreConfig(&save_config, config); + save_config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&save_config, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } /* Set LC_CTYPE to the user preferred locale */ if (config->configure_locale) { @@ -923,7 +976,12 @@ PyStatus _PyPreConfig_Write(const PyPreConfig *src_config) { PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (_PyRuntime.core_initialized) { /* bpo-34008: Calling this functions after Py_Initialize() ignores diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index eed583a4fdbc..ea0d7a5ee2b9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -735,7 +735,12 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) runtime->preinitializing = 1; PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } status = _PyPreConfig_Read(&config, args); if (_PyStatus_EXCEPTION(status)) { @@ -794,7 +799,12 @@ _Py_PreInitializeFromConfig(const PyConfig *config, } PyPreConfig preconfig; - _PyPreConfig_InitFromConfig(&preconfig, config); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromConfig(&preconfig, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (!config->parse_argv) { return Py_PreInitialize(&preconfig); @@ -842,7 +852,12 @@ pyinit_core(_PyRuntimeState *runtime, } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } status = _PyConfig_Copy(&config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -1064,7 +1079,13 @@ Py_InitializeEx(int install_sigs) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } + config.install_signal_handlers = install_sigs; status = Py_InitializeFromConfig(&config); diff --git a/Python/pystate.c b/Python/pystate.c index 7dd8b7f866b5..0f0cb2299557 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -60,7 +60,12 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); - PyPreConfig_InitPythonConfig(&runtime->preconfig); + + runtime->preconfig.struct_size = sizeof(PyPreConfig); + PyStatus status = PyPreConfig_InitPythonConfig(&runtime->preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } runtime->gilstate.check_enabled = 1; @@ -204,6 +209,7 @@ PyInterpreterState_New(void) memset(interp, 0, sizeof(*interp)); interp->id_refcount = -1; + interp->config.struct_size = sizeof(PyConfig); PyStatus status = PyConfig_InitPythonConfig(&interp->config); if (_PyStatus_EXCEPTION(status)) { /* Don't report status to caller: PyConfig_InitPythonConfig()

1 0