Mailman 3 September 2019 - Python-checkins

bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079) (GH-16464)
by Gregory P. Smith Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/36c6fa968016a46a39c3cdbd0a17ea5490… commit: 36c6fa968016a46a39c3cdbd0a17ea5490dfa343 branch: 3.8 author: Gregory P. Smith <greg(a)krypto.org> committer: GitHub <noreply(a)github.com> date: 2019-09-28T08:22:00-07:00 summary: bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079) (GH-16464) Document that lnotab can contain invalid bytecode offsets (because of terrible reasons that are difficult to fix). Make dis.findlinestarts() … [View More]ignore invalid offsets in lnotab. All other uses of lnotab in CPython (various reimplementations of addr2line or line2addr in Python, C and gdb) already ignore this, because they take an address to look for, instead. Add tests for the result of dis.findlinestarts() on wacky constructs in test_peepholer.py, because it's the easiest place to add them. (cherry picked from commit c8165036f374cd2ee64d4314eeb2514f7acb5026) files: A Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst M Lib/dis.py M Lib/test/test_peepholer.py M Objects/lnotab_notes.txt diff --git a/Lib/dis.py b/Lib/dis.py index a25fb2b41764..10e5f7fb08ab 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -454,6 +454,7 @@ def findlinestarts(code): """ byte_increments = code.co_lnotab[0::2] line_increments = code.co_lnotab[1::2] + bytecode_len = len(code.co_code) lastlineno = None lineno = code.co_firstlineno @@ -464,6 +465,10 @@ def findlinestarts(code): yield (addr, lineno) lastlineno = lineno addr += byte_incr + if addr >= bytecode_len: + # The rest of the lnotab byte offsets are past the end of + # the bytecode, so the lines were optimized away. + return if line_incr >= 0x80: # line_increments is an array of 8-bit signed integers line_incr -= 0x100 diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 9c206d1d09c0..01c00369156b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -40,6 +40,20 @@ def check_jump_targets(self, code): self.fail(f'{instr.opname} at {instr.offset} ' f'jumps to {tgt.opname} at {tgt.offset}') + def check_lnotab(self, code): + "Check that the lnotab byte offsets are sensible." + code = dis._get_code_object(code) + lnotab = list(dis.findlinestarts(code)) + # Don't bother checking if the line info is sensible, because + # most of the line info we can get at comes from lnotab. + min_bytecode = min(t[0] for t in lnotab) + max_bytecode = max(t[0] for t in lnotab) + self.assertGreaterEqual(min_bytecode, 0) + self.assertLess(max_bytecode, len(code.co_code)) + # This could conceivably test more (and probably should, as there + # aren't very many tests of lnotab), if peepholer wasn't scheduled + # to be replaced anyway. + def test_unot(self): # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' def unot(x): @@ -48,6 +62,7 @@ def unot(x): self.assertNotInBytecode(unot, 'UNARY_NOT') self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') + self.check_lnotab(unot) def test_elim_inversion_of_is_or_in(self): for line, cmp_op in ( @@ -58,6 +73,7 @@ def test_elim_inversion_of_is_or_in(self): ): code = compile(line, '', 'single') self.assertInBytecode(code, 'COMPARE_OP', cmp_op) + self.check_lnotab(code) def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False @@ -75,6 +91,7 @@ def h(): for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + self.check_lnotab(func) def f(): 'Adding a docstring made this test fail in Py2.5.0' @@ -82,6 +99,7 @@ def f(): self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) + self.check_lnotab(f) def test_while_one(self): # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx @@ -93,6 +111,7 @@ def f(): self.assertNotInBytecode(f, elem) for elem in ('JUMP_ABSOLUTE',): self.assertInBytecode(f, elem) + self.check_lnotab(f) def test_pack_unpack(self): for line, elem in ( @@ -104,6 +123,7 @@ def test_pack_unpack(self): self.assertInBytecode(code, elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') self.assertNotInBytecode(code, 'UNPACK_TUPLE') + self.check_lnotab(code) def test_folding_of_tuples_of_constants(self): for line, elem in ( @@ -116,6 +136,7 @@ def test_folding_of_tuples_of_constants(self): code = compile(line,'','single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.check_lnotab(code) # Long tuples should be folded too. code = compile(repr(tuple(range(10000))),'','single') @@ -124,6 +145,7 @@ def test_folding_of_tuples_of_constants(self): load_consts = [instr for instr in dis.get_instructions(code) if instr.opname == 'LOAD_CONST'] self.assertEqual(len(load_consts), 2) + self.check_lnotab(code) # Bug 1053819: Tuple of constants misidentified when presented with: # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . @@ -141,6 +163,7 @@ def crater(): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ],) + self.check_lnotab(crater) def test_folding_of_lists_of_constants(self): for line, elem in ( @@ -153,6 +176,7 @@ def test_folding_of_lists_of_constants(self): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_LIST') + self.check_lnotab(code) def test_folding_of_sets_of_constants(self): for line, elem in ( @@ -166,6 +190,7 @@ def test_folding_of_sets_of_constants(self): code = compile(line, '', 'single') self.assertNotInBytecode(code, 'BUILD_SET') self.assertInBytecode(code, 'LOAD_CONST', elem) + self.check_lnotab(code) # Ensure that the resulting code actually works: def f(a): @@ -176,9 +201,11 @@ def g(a): self.assertTrue(f(3)) self.assertTrue(not f(4)) + self.check_lnotab(f) self.assertTrue(not g(3)) self.assertTrue(g(4)) + self.check_lnotab(g) def test_folding_of_binops_on_constants(self): @@ -203,41 +230,50 @@ def test_folding_of_binops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('BINARY_')) + self.check_lnotab(code) # Verify that unfoldables are skipped code = compile('a=2+"b"', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 2) self.assertInBytecode(code, 'LOAD_CONST', 'b') + self.check_lnotab(code) # Verify that large sequences do not result from folding code = compile('a="x"*10000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 10000) self.assertNotIn("x"*10000, code.co_consts) + self.check_lnotab(code) code = compile('a=1<<1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(1<<1000, code.co_consts) + self.check_lnotab(code) code = compile('a=2**1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(2**1000, code.co_consts) + self.check_lnotab(code) def test_binary_subscr_on_unicode(self): # valid code get optimized code = compile('"foo"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 'f') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) code = compile('"\u0061\uffff"[1]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\uffff') self.assertNotInBytecode(code,'BINARY_SUBSCR') + self.check_lnotab(code) # With PEP 393, non-BMP char get optimized code = compile('"\U00012345"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) # invalid code doesn't get optimized # out of range code = compile('"fuu"[10]', '', 'single') self.assertInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) def test_folding_of_unaryops_on_constants(self): for line, elem in ( @@ -252,13 +288,15 @@ def test_folding_of_unaryops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(code) # Check that -0.0 works after marshaling def negzero(): return -(1.0-1.0) - for instr in dis.get_instructions(code): + for instr in dis.get_instructions(negzero): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(negzero) # Verify that unfoldables are skipped for line, elem, opname in ( @@ -268,6 +306,7 @@ def negzero(): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertInBytecode(code, opname) + self.check_lnotab(code) def test_elim_extra_return(self): # RETURN LOAD_CONST None RETURN --> RETURN @@ -277,6 +316,7 @@ def f(x): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 1) + self.check_lnotab(f) def test_elim_jump_to_return(self): # JUMP_FORWARD to RETURN --> RETURN @@ -290,6 +330,7 @@ def f(cond, true_value, false_value): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 2) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump(self): # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump @@ -302,6 +343,7 @@ def f(): else: baz() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump2(self): # POP_JUMP_IF_FALSE to JUMP_ABSOLUTE --> POP_JUMP_IF_FALSE to non-jump @@ -312,6 +354,7 @@ def f(): or d): a = foo() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump3(self): # Intentionally use two-line expressions to test issue37213. @@ -320,18 +363,21 @@ def f(a, b, c): return ((a and b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_FALSE_OR_POP'), 2) # JUMP_IF_TRUE_OR_POP to JUMP_IF_TRUE_OR_POP --> JUMP_IF_TRUE_OR_POP to non-jump def f(a, b, c): return ((a or b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_TRUE_OR_POP'), 2) # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump def f(a, b, c): return ((a and b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_FALSE') @@ -340,6 +386,7 @@ def f(a, b, c): return ((a or b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_TRUE') @@ -360,6 +407,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 6) + self.check_lnotab(f) def test_elim_jump_after_return2(self): # Eliminate dead code: jumps immediately after returns can't be reached @@ -374,6 +422,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 2) + self.check_lnotab(f) def test_make_function_doesnt_bail(self): def f(): @@ -381,6 +430,7 @@ def g()->1+1: pass return g self.assertNotInBytecode(f, 'BINARY_ADD') + self.check_lnotab(f) def test_constant_folding(self): # Issue #11244: aggressive constant folding. @@ -401,17 +451,20 @@ def test_constant_folding(self): self.assertFalse(instr.opname.startswith('UNARY_')) self.assertFalse(instr.opname.startswith('BINARY_')) self.assertFalse(instr.opname.startswith('BUILD_')) + self.check_lnotab(code) def test_in_literal_list(self): def containtest(): return x in [a, b] self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + self.check_lnotab(containtest) def test_iterate_literal_list(self): def forloop(): for x in [a, b]: pass self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + self.check_lnotab(forloop) def test_condition_with_binop_with_bools(self): def f(): @@ -419,6 +472,28 @@ def f(): return 1 return 0 self.assertEqual(f(), 1) + self.check_lnotab(f) + + def test_if_with_if_expression(self): # XXX does this belong in 3.8? + # Check bpo-37289 + def f(x): + if (True if x else False): + return True + return False + self.assertTrue(f(True)) + self.check_lnotab(f) + + def test_trailing_nops(self): + # Check the lnotab of a function that even after trivial + # optimization has trailing nops, which the lnotab adjustment has to + # handle properly (bpo-38115). + def f(x): + while 1: + return 3 + while 1: + return 5 + return 6 + self.check_lnotab(f) class TestBuglets(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst new file mode 100644 index 000000000000..5119c0546e36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst @@ -0,0 +1 @@ +Fix a bug in dis.findlinestarts() where it would return invalid bytecode offsets. Document that a code object's co_lnotab can contain invalid bytecode offsets. \ No newline at end of file diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 3dab2b986616..71a297971828 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -3,7 +3,9 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes disguised as a Python bytes object. It is used to map bytecode offsets to source code line #s for tracebacks and to identify line number boundaries for -line tracing. +line tracing. Because of internals of the peephole optimizer, it's possible +for lnotab to contain bytecode offsets that are no longer valid (for example +if the optimizer removed the last line in a function). The array is conceptually a compressed list of (bytecode offset increment, line number increment) [View Less]

1 0

bpo-37408: Precise that Tarfile "format" argument only concerns writing. (GH-14389)
by Gregory P. Smith Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/c5a7e0ce194c0eafe82eb3e43188101239… commit: c5a7e0ce194c0eafe82eb3e431881012398e7d46 branch: master author: Pascal Chambon <pythoniks(a)gmail.com> committer: Gregory P. Smith <greg(a)krypto.org> date: 2019-09-28T08:04:44-07:00 summary: bpo-37408: Precise that Tarfile "format" argument only concerns writing. (GH-14389) files: M Doc/library/tarfile.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f25af8ca6a33..… [View More]

1 0

bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079)
by Gregory P. Smith Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/c8165036f374cd2ee64d4314eeb2514f7a… commit: c8165036f374cd2ee64d4314eeb2514f7acb5026 branch: master author: T. Wouters <thomas(a)python.org> committer: Gregory P. Smith <greg(a)krypto.org> date: 2019-09-28T07:49:15-07:00 summary: bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079) Document that lnotab can contain invalid bytecode offsets (because of terrible reasons that are difficult to fix). Make dis.findlinestarts() ignore … [View More]invalid offsets in lnotab. All other uses of lnotab in CPython (various reimplementations of addr2line or line2addr in Python, C and gdb) already ignore this, because they take an address to look for, instead. Add tests for the result of dis.findlinestarts() on wacky constructs in test_peepholer.py, because it's the easiest place to add them. files: A Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst M Lib/dis.py M Lib/test/test_peepholer.py M Objects/lnotab_notes.txt diff --git a/Lib/dis.py b/Lib/dis.py index a25fb2b41764..10e5f7fb08ab 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -454,6 +454,7 @@ def findlinestarts(code): """ byte_increments = code.co_lnotab[0::2] line_increments = code.co_lnotab[1::2] + bytecode_len = len(code.co_code) lastlineno = None lineno = code.co_firstlineno @@ -464,6 +465,10 @@ def findlinestarts(code): yield (addr, lineno) lastlineno = lineno addr += byte_incr + if addr >= bytecode_len: + # The rest of the lnotab byte offsets are past the end of + # the bytecode, so the lines were optimized away. + return if line_incr >= 0x80: # line_increments is an array of 8-bit signed integers line_incr -= 0x100 diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 47dee33076c5..23cc36c60537 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -40,6 +40,20 @@ def check_jump_targets(self, code): self.fail(f'{instr.opname} at {instr.offset} ' f'jumps to {tgt.opname} at {tgt.offset}') + def check_lnotab(self, code): + "Check that the lnotab byte offsets are sensible." + code = dis._get_code_object(code) + lnotab = list(dis.findlinestarts(code)) + # Don't bother checking if the line info is sensible, because + # most of the line info we can get at comes from lnotab. + min_bytecode = min(t[0] for t in lnotab) + max_bytecode = max(t[0] for t in lnotab) + self.assertGreaterEqual(min_bytecode, 0) + self.assertLess(max_bytecode, len(code.co_code)) + # This could conceivably test more (and probably should, as there + # aren't very many tests of lnotab), if peepholer wasn't scheduled + # to be replaced anyway. + def test_unot(self): # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' def unot(x): @@ -48,6 +62,7 @@ def unot(x): self.assertNotInBytecode(unot, 'UNARY_NOT') self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') + self.check_lnotab(unot) def test_elim_inversion_of_is_or_in(self): for line, cmp_op in ( @@ -58,6 +73,7 @@ def test_elim_inversion_of_is_or_in(self): ): code = compile(line, '', 'single') self.assertInBytecode(code, 'COMPARE_OP', cmp_op) + self.check_lnotab(code) def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False @@ -75,6 +91,7 @@ def h(): for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + self.check_lnotab(func) def f(): 'Adding a docstring made this test fail in Py2.5.0' @@ -82,6 +99,7 @@ def f(): self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) + self.check_lnotab(f) def test_while_one(self): # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx @@ -93,6 +111,7 @@ def f(): self.assertNotInBytecode(f, elem) for elem in ('JUMP_ABSOLUTE',): self.assertInBytecode(f, elem) + self.check_lnotab(f) def test_pack_unpack(self): for line, elem in ( @@ -104,6 +123,7 @@ def test_pack_unpack(self): self.assertInBytecode(code, elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') self.assertNotInBytecode(code, 'UNPACK_TUPLE') + self.check_lnotab(code) def test_folding_of_tuples_of_constants(self): for line, elem in ( @@ -116,6 +136,7 @@ def test_folding_of_tuples_of_constants(self): code = compile(line,'','single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.check_lnotab(code) # Long tuples should be folded too. code = compile(repr(tuple(range(10000))),'','single') @@ -124,6 +145,7 @@ def test_folding_of_tuples_of_constants(self): load_consts = [instr for instr in dis.get_instructions(code) if instr.opname == 'LOAD_CONST'] self.assertEqual(len(load_consts), 2) + self.check_lnotab(code) # Bug 1053819: Tuple of constants misidentified when presented with: # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . @@ -141,6 +163,7 @@ def crater(): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ],) + self.check_lnotab(crater) def test_folding_of_lists_of_constants(self): for line, elem in ( @@ -153,6 +176,7 @@ def test_folding_of_lists_of_constants(self): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertNotInBytecode(code, 'BUILD_LIST') + self.check_lnotab(code) def test_folding_of_sets_of_constants(self): for line, elem in ( @@ -166,6 +190,7 @@ def test_folding_of_sets_of_constants(self): code = compile(line, '', 'single') self.assertNotInBytecode(code, 'BUILD_SET') self.assertInBytecode(code, 'LOAD_CONST', elem) + self.check_lnotab(code) # Ensure that the resulting code actually works: def f(a): @@ -176,9 +201,11 @@ def g(a): self.assertTrue(f(3)) self.assertTrue(not f(4)) + self.check_lnotab(f) self.assertTrue(not g(3)) self.assertTrue(g(4)) + self.check_lnotab(g) def test_folding_of_binops_on_constants(self): @@ -203,41 +230,50 @@ def test_folding_of_binops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('BINARY_')) + self.check_lnotab(code) # Verify that unfoldables are skipped code = compile('a=2+"b"', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 2) self.assertInBytecode(code, 'LOAD_CONST', 'b') + self.check_lnotab(code) # Verify that large sequences do not result from folding code = compile('a="x"*10000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 10000) self.assertNotIn("x"*10000, code.co_consts) + self.check_lnotab(code) code = compile('a=1<<1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(1<<1000, code.co_consts) + self.check_lnotab(code) code = compile('a=2**1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(2**1000, code.co_consts) + self.check_lnotab(code) def test_binary_subscr_on_unicode(self): # valid code get optimized code = compile('"foo"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 'f') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) code = compile('"\u0061\uffff"[1]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\uffff') self.assertNotInBytecode(code,'BINARY_SUBSCR') + self.check_lnotab(code) # With PEP 393, non-BMP char get optimized code = compile('"\U00012345"[0]', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') self.assertNotInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) # invalid code doesn't get optimized # out of range code = compile('"fuu"[10]', '', 'single') self.assertInBytecode(code, 'BINARY_SUBSCR') + self.check_lnotab(code) def test_folding_of_unaryops_on_constants(self): for line, elem in ( @@ -252,13 +288,15 @@ def test_folding_of_unaryops_on_constants(self): self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(code) # Check that -0.0 works after marshaling def negzero(): return -(1.0-1.0) - for instr in dis.get_instructions(code): + for instr in dis.get_instructions(negzero): self.assertFalse(instr.opname.startswith('UNARY_')) + self.check_lnotab(negzero) # Verify that unfoldables are skipped for line, elem, opname in ( @@ -268,6 +306,7 @@ def negzero(): code = compile(line, '', 'single') self.assertInBytecode(code, 'LOAD_CONST', elem) self.assertInBytecode(code, opname) + self.check_lnotab(code) def test_elim_extra_return(self): # RETURN LOAD_CONST None RETURN --> RETURN @@ -277,6 +316,7 @@ def f(x): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 1) + self.check_lnotab(f) def test_elim_jump_to_return(self): # JUMP_FORWARD to RETURN --> RETURN @@ -290,6 +330,7 @@ def f(cond, true_value, false_value): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertEqual(len(returns), 2) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump(self): # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump @@ -302,6 +343,7 @@ def f(): else: baz() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump2(self): # POP_JUMP_IF_FALSE to JUMP_ABSOLUTE --> POP_JUMP_IF_FALSE to non-jump @@ -312,6 +354,7 @@ def f(): or d): a = foo() self.check_jump_targets(f) + self.check_lnotab(f) def test_elim_jump_to_uncond_jump3(self): # Intentionally use two-line expressions to test issue37213. @@ -320,18 +363,21 @@ def f(a, b, c): return ((a and b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_FALSE_OR_POP'), 2) # JUMP_IF_TRUE_OR_POP to JUMP_IF_TRUE_OR_POP --> JUMP_IF_TRUE_OR_POP to non-jump def f(a, b, c): return ((a or b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertEqual(count_instr_recursively(f, 'JUMP_IF_TRUE_OR_POP'), 2) # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump def f(a, b, c): return ((a and b) or c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_FALSE') @@ -340,6 +386,7 @@ def f(a, b, c): return ((a or b) and c) self.check_jump_targets(f) + self.check_lnotab(f) self.assertNotInBytecode(f, 'JUMP_IF_TRUE_OR_POP') self.assertInBytecode(f, 'JUMP_IF_FALSE_OR_POP') self.assertInBytecode(f, 'POP_JUMP_IF_TRUE') @@ -360,6 +407,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 6) + self.check_lnotab(f) def test_elim_jump_after_return2(self): # Eliminate dead code: jumps immediately after returns can't be reached @@ -374,6 +422,7 @@ def f(cond1, cond2): returns = [instr for instr in dis.get_instructions(f) if instr.opname == 'RETURN_VALUE'] self.assertLessEqual(len(returns), 2) + self.check_lnotab(f) def test_make_function_doesnt_bail(self): def f(): @@ -381,6 +430,7 @@ def g()->1+1: pass return g self.assertNotInBytecode(f, 'BINARY_ADD') + self.check_lnotab(f) def test_constant_folding(self): # Issue #11244: aggressive constant folding. @@ -401,17 +451,20 @@ def test_constant_folding(self): self.assertFalse(instr.opname.startswith('UNARY_')) self.assertFalse(instr.opname.startswith('BINARY_')) self.assertFalse(instr.opname.startswith('BUILD_')) + self.check_lnotab(code) def test_in_literal_list(self): def containtest(): return x in [a, b] self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + self.check_lnotab(containtest) def test_iterate_literal_list(self): def forloop(): for x in [a, b]: pass self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + self.check_lnotab(forloop) def test_condition_with_binop_with_bools(self): def f(): @@ -419,6 +472,7 @@ def f(): return 1 return 0 self.assertEqual(f(), 1) + self.check_lnotab(f) def test_if_with_if_expression(self): # Check bpo-37289 @@ -427,6 +481,19 @@ def f(x): return True return False self.assertTrue(f(True)) + self.check_lnotab(f) + + def test_trailing_nops(self): + # Check the lnotab of a function that even after trivial + # optimization has trailing nops, which the lnotab adjustment has to + # handle properly (bpo-38115). + def f(x): + while 1: + return 3 + while 1: + return 5 + return 6 + self.check_lnotab(f) class TestBuglets(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst new file mode 100644 index 000000000000..5119c0546e36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst @@ -0,0 +1 @@ +Fix a bug in dis.findlinestarts() where it would return invalid bytecode offsets. Document that a code object's co_lnotab can contain invalid bytecode offsets. \ No newline at end of file diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 3dab2b986616..71a297971828 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -3,7 +3,9 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes disguised as a Python bytes object. It is used to map bytecode offsets to source code line #s for tracebacks and to identify line number boundaries for -line tracing. +line tracing. Because of internals of the peephole optimizer, it's possible +for lnotab to contain bytecode offsets that are no longer valid (for example +if the optimizer removed the last line in a function). The array is conceptually a compressed list of (bytecode offset increment, line number increment) [View Less]

1 0

bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448)
by Miss Islington (bot) Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/8f478b489ae11633d2609dff0ef21d0e1a… commit: 8f478b489ae11633d2609dff0ef21d0e1a857417 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-09-28T07:23:34-07:00 summary: bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) * bpo-38216: Allow bypassing input validation * bpo-36274: Also allow the … [View More]URL encoding to be overridden. * bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL. * Call with skip_host to avoid tripping on the host checking in the URL. * Remove obsolete comment. * Make _prepare_path_encoding its own attr. This makes overriding just that simpler. Also, don't use the := operator to make backporting easier. * Add a news entry. * _prepare_path_encoding -> _encode_prepared_path() * Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path. (cherry picked from commit 7774d7831e8809795c64ce27f7df52674581d298) Co-authored-by: Jason R. Coombs <jaraco(a)jaraco.com> files: A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst M Lib/http/client.py M Lib/test/test_httplib.py diff --git a/Lib/http/client.py b/Lib/http/client.py index f61267e108a5..33a434733f8a 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1085,18 +1085,15 @@ def putrequest(self, method, url, skip_host=False, else: raise CannotSendRequest(self.__state) - # Save the method we use, we need it later in the response phase + # Save the method for use later in the response phase self._method = method - if not url: - url = '/' - # Prevent CVE-2019-9740. - if match := _contains_disallowed_url_pchar_re.search(url): - raise InvalidURL(f"URL can't contain control characters. {url!r} " - f"(found at least {match.group()!r})") + + url = url or '/' + self._validate_path(url) + request = '%s %s %s' % (method, url, self._http_vsn_str) - # Non-ASCII characters should have been eliminated earlier - self._output(request.encode('ascii')) + self._output(self._encode_request(request)) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance @@ -1174,6 +1171,18 @@ def putrequest(self, method, url, skip_host=False, # For HTTP/1.0, the server will assume "not chunked" pass + def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 1121224da443..e1aa41421feb 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1155,6 +1155,34 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_putrequest_override_validation(self): + """ + It should be possible to override the default validation + behavior in putrequest (bpo-38216). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_path(self, url): + pass + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/\x00') + + def test_putrequest_override_encoding(self): + """ + It should be possible to override the default encoding + to transmit bytes in another encoding even if invalid + (bpo-36274). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _encode_request(self, str_url): + return str_url.encode('utf-8') + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/☃') + + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1279,6 +1307,7 @@ def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst new file mode 100644 index 000000000000..ac8e2b042d92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst @@ -0,0 +1,4 @@ +Allow the rare code that wants to send invalid http requests from the +`http.client` library a way to do so. The fixes for bpo-30458 led to +breakage for some projects that were relying on this ability to test their +own behavior in the face of bad requests. [View Less]

1 0

[3.7] bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) (GH-16461)
by Jason R. Coombs Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/80dd66ac278ecbabbf843526e3a56f5031… commit: 80dd66ac278ecbabbf843526e3a56f5031da9562 branch: 3.7 author: Jason R. Coombs <jaraco(a)jaraco.com> committer: GitHub <noreply(a)github.com> date: 2019-09-28T09:15:05-04:00 summary: [3.7] bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) (GH-16461) * bpo-38216: Allow bypassing input validation * bpo-36274: Also allow the URL encoding to be … [View More]overridden. * bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL. * Call with skip_host to avoid tripping on the host checking in the URL. * Remove obsolete comment. * Make _prepare_path_encoding its own attr. This makes overriding just that simpler. Also, don't use the := operator to make backporting easier. * Add a news entry. * _prepare_path_encoding -> _encode_prepared_path() * Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path.. (cherry picked from commit 7774d7831e8809795c64ce27f7df52674581d298) Co-authored-by: Jason R. Coombs <jaraco(a)jaraco.com> files: A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst M Lib/http/client.py M Lib/test/test_httplib.py diff --git a/Lib/http/client.py b/Lib/http/client.py index dd23edcd597c..f7a183c77403 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1107,19 +1107,15 @@ def putrequest(self, method, url, skip_host=False, else: raise CannotSendRequest(self.__state) - # Save the method we use, we need it later in the response phase + # Save the method for use later in the response phase self._method = method - if not url: - url = '/' - # Prevent CVE-2019-9740. - match = _contains_disallowed_url_pchar_re.search(url) - if match: - raise InvalidURL(f"URL can't contain control characters. {url!r} " - f"(found at least {match.group()!r})") + + url = url or '/' + self._validate_path(url) + request = '%s %s %s' % (method, url, self._http_vsn_str) - # Non-ASCII characters should have been eliminated earlier - self._output(request.encode('ascii')) + self._output(self._encode_request(request)) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance @@ -1197,6 +1193,18 @@ def putrequest(self, method, url, skip_host=False, # For HTTP/1.0, the server will assume "not chunked" pass + def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index c4246671586f..9facb2e36017 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1157,6 +1157,34 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_putrequest_override_validation(self): + """ + It should be possible to override the default validation + behavior in putrequest (bpo-38216). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_path(self, url): + pass + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/\x00') + + def test_putrequest_override_encoding(self): + """ + It should be possible to override the default encoding + to transmit bytes in another encoding even if invalid + (bpo-36274). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _encode_request(self, str_url): + return str_url.encode('utf-8') + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/☃') + + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1281,6 +1309,7 @@ def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst new file mode 100644 index 000000000000..ac8e2b042d92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst @@ -0,0 +1,4 @@ +Allow the rare code that wants to send invalid http requests from the +`http.client` library a way to do so. The fixes for bpo-30458 led to +breakage for some projects that were relying on this ability to test their +own behavior in the face of bad requests. [View Less]

1 0

bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448)
by Jason R. Coombs Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/7774d7831e8809795c64ce27f7df526745… commit: 7774d7831e8809795c64ce27f7df52674581d298 branch: master author: Jason R. Coombs <jaraco(a)jaraco.com> committer: GitHub <noreply(a)github.com> date: 2019-09-28T08:32:01-04:00 summary: bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) * bpo-38216: Allow bypassing input validation * bpo-36274: Also allow the URL encoding to be overridden. * … [View More]bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL. * Call with skip_host to avoid tripping on the host checking in the URL. * Remove obsolete comment. * Make _prepare_path_encoding its own attr. This makes overriding just that simpler. Also, don't use the := operator to make backporting easier. * Add a news entry. * _prepare_path_encoding -> _encode_prepared_path() * Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path. files: A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst M Lib/http/client.py M Lib/test/test_httplib.py diff --git a/Lib/http/client.py b/Lib/http/client.py index f61267e108a5..33a434733f8a 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1085,18 +1085,15 @@ def putrequest(self, method, url, skip_host=False, else: raise CannotSendRequest(self.__state) - # Save the method we use, we need it later in the response phase + # Save the method for use later in the response phase self._method = method - if not url: - url = '/' - # Prevent CVE-2019-9740. - if match := _contains_disallowed_url_pchar_re.search(url): - raise InvalidURL(f"URL can't contain control characters. {url!r} " - f"(found at least {match.group()!r})") + + url = url or '/' + self._validate_path(url) + request = '%s %s %s' % (method, url, self._http_vsn_str) - # Non-ASCII characters should have been eliminated earlier - self._output(request.encode('ascii')) + self._output(self._encode_request(request)) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance @@ -1174,6 +1171,18 @@ def putrequest(self, method, url, skip_host=False, # For HTTP/1.0, the server will assume "not chunked" pass + def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 1121224da443..e1aa41421feb 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1155,6 +1155,34 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_putrequest_override_validation(self): + """ + It should be possible to override the default validation + behavior in putrequest (bpo-38216). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_path(self, url): + pass + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/\x00') + + def test_putrequest_override_encoding(self): + """ + It should be possible to override the default encoding + to transmit bytes in another encoding even if invalid + (bpo-36274). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _encode_request(self, str_url): + return str_url.encode('utf-8') + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/☃') + + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1279,6 +1307,7 @@ def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst new file mode 100644 index 000000000000..ac8e2b042d92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst @@ -0,0 +1,4 @@ +Allow the rare code that wants to send invalid http requests from the +`http.client` library a way to do so. The fixes for bpo-30458 led to +breakage for some projects that were relying on this ability to test their +own behavior in the face of bad requests. [View Less]

1 0

bpo-38243, xmlrpc.server: Escape the server_title (GH-16373) (GH-16441)
by Ned Deily Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/1698cacfb924d1df452e78d11a4bf81ae7… commit: 1698cacfb924d1df452e78d11a4bf81ae7777389 branch: 3.6 author: Victor Stinner <vstinner(a)redhat.com> committer: Ned Deily <nad(a)python.org> date: 2019-09-28T03:33:00-04:00 summary: bpo-38243, xmlrpc.server: Escape the server_title (GH-16373) (GH-16441) Escape the server title of xmlrpc.server.DocXMLRPCServer when rendering the document page as HTML. (cherry picked from commit … [View More]e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa) files: A Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst M Lib/test/test_docxmlrpc.py M Lib/xmlrpc/server.py diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 00903337c07c..d2adb21af0fb 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -1,5 +1,6 @@ from xmlrpc.server import DocXMLRPCServer import http.client +import re import sys from test import support threading = support.import_module('threading') @@ -193,6 +194,21 @@ def test_annotations(self): b'method_annotation</strong></a>(x: bytes)</dt></dl>'), response.read()) + def test_server_title_escape(self): + # bpo-38243: Ensure that the server title and documentation + # are escaped for HTML. + self.serv.set_server_title('test_title<script>') + self.serv.set_server_documentation('test_documentation<script>') + self.assertEqual('test_title<script>', self.serv.server_title) + self.assertEqual('test_documentation<script>', + self.serv.server_documentation) + + generated = self.serv.generate_html_documentation() + title = re.search(r'<title>(.+?)</title>', generated).group() + documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group() + self.assertEqual('<title>Python: test_title<script></title>', title) + self.assertEqual('<p><tt>test_documentation<script></tt></p>', documentation) + if __name__ == '__main__': unittest.main() diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 3e0dca027f06..efe593748968 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -106,6 +106,7 @@ def export_add(self, x, y): from xmlrpc.client import Fault, dumps, loads, gzip_encode, gzip_decode from http.server import BaseHTTPRequestHandler +import html import http.server import socketserver import sys @@ -904,7 +905,7 @@ def generate_html_documentation(self): methods ) - return documenter.page(self.server_title, documentation) + return documenter.page(html.escape(self.server_title), documentation) class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): """XML-RPC and documentation request handler class. diff --git a/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst new file mode 100644 index 000000000000..98d7be129573 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst @@ -0,0 +1,3 @@ +Escape the server title of :class:`xmlrpc.server.DocXMLRPCServer` +when rendering the document page as HTML. +(Contributed by Dong-hee Na in :issue:`38243`.) [View Less]

1 0

(no subject)
by Jesús Cea Sept. 28, 2019

Sept. 28, 2019

To: python-checkins(a)python.org Subject: [2.7] bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (GH-16446). (#16454) Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 https://github.com/python/cpython/commit/598f676880662fb453ff98fda42b7b7068… be32 commit: 598f676880662fb453ff98fda42b7b7068e5be32 branch: 2.7 author: Jes=C3=BAs Cea <jcea(a)jcea.es> committer: GitHub <noreply(a)github.com> date: 2019-09-28T05:… [View More]

1 0

bpo-38304: Add PyConfig.struct_size (GH-16451) (GH-16453)
by Victor Stinner Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/6e128382b3e7f83d91d5e73d83a051ff5f… commit: 6e128382b3e7f83d91d5e73d83a051ff5fb6469c branch: 3.8 author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-09-28T04:50:43+02:00 summary: bpo-38304: Add PyConfig.struct_size (GH-16451) (GH-16453) Add a new struct_size field to PyPreConfig and PyConfig structures to allow to modify these structures in the future without breaking the backward compatibility.… [View More] * Replace private _config_version field with public struct_size field in PyPreConfig and PyConfig. * Public PyPreConfig_InitIsolatedConfig() and PyPreConfig_InitPythonConfig() return type becomes PyStatus, instead of void. * Internal _PyConfig_InitCompatConfig(), _PyPreConfig_InitCompatConfig(), _PyPreConfig_InitFromConfig(), _PyPreConfig_InitFromPreConfig() return type becomes PyStatus, instead of void. * Remove _Py_CONFIG_VERSION * Update the Initialization Configuration documentation. (cherry picked from commit 441b10cf2855955c86565f8d59e72c2efc0f0a57) files: A Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst M Doc/c-api/init_config.rst M Include/cpython/initconfig.h M Include/internal/pycore_initconfig.h M Modules/main.c M Programs/_freeze_importlib.c M Programs/_testembed.c M Python/frozenmain.c M Python/initconfig.c M Python/pathconfig.c M Python/preconfig.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0c3c725c841a..58e417441d13 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -194,18 +194,25 @@ PyPreConfig * Configure the LC_CTYPE locale * Set the UTF-8 mode + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyPreConfig)``. + Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Python Configuration <init-python-config>`. - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Isolated Configuration <init-isolated-conf>`. + The caller of these functions is responsible to handle exceptions (error or + exit) using :c:func:`PyStatus_Exception` and + :c:func:`Py_ExitStatusException`. + Structure fields: .. c:member:: int allocator @@ -267,6 +274,13 @@ PyPreConfig same way the regular Python parses command line arguments: see :ref:`Command Line Arguments <using-on-cmdline>`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyPreConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int use_environment See :c:member:`PyConfig.use_environment`. @@ -316,12 +330,18 @@ the preinitialization. Example using the preinitialization to enable the UTF-8 Mode:: + PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.utf8_mode = 1; - PyStatus status = Py_PreInitialize(&preconfig); + status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); } @@ -340,6 +360,9 @@ PyConfig Structure containing most parameters to configure Python. + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyConfig)``. + Structure methods: .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config) @@ -656,6 +679,13 @@ PyConfig Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int tracemalloc If non-zero, call :func:`tracemalloc.start` at startup. @@ -718,6 +748,7 @@ Example setting the program name:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -750,6 +781,7 @@ configuration, and then override some parameters:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -835,8 +867,9 @@ Example of customized Python always running in isolated mode:: int main(int argc, char **argv) { - PyConfig config; PyStatus status; + PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1028,6 +1061,7 @@ phases:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 047f511f3104..d5effb8397fc 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -45,8 +45,10 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list, /* --- PyPreConfig ----------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyPreConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ /* Parse Py_PreInitializeFromBytesArgs() arguments? @@ -122,15 +124,17 @@ typedef struct { int allocator; } PyPreConfig; -PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config); -PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); /* --- PyConfig ---------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ int isolated; /* Isolated mode? see PyPreConfig.isolated */ @@ -403,7 +407,6 @@ typedef struct { /* If equal to 0, stop Python initialization before the "main" phase */ int _init_main; - } PyConfig; PyAPI_FUNC(PyStatus) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index a7c1994eca16..dcdb61695cab 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -40,6 +40,8 @@ extern "C" { (err._type == _PyStatus_TYPE_EXIT) #define _PyStatus_EXCEPTION(err) \ (err._type != _PyStatus_TYPE_OK) +#define _PyStatus_UPDATE_FUNC(err) \ + do { err.func = _PyStatus_GET_FUNC(); } while (0) /* --- PyWideStringList ------------------------------------------------ */ @@ -118,11 +120,11 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline, /* --- PyPreConfig ----------------------------------------------- */ -PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); -extern void _PyPreConfig_InitFromConfig( +PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); +extern PyStatus _PyPreConfig_InitFromConfig( PyPreConfig *preconfig, const PyConfig *config); -extern void _PyPreConfig_InitFromPreConfig( +extern PyStatus _PyPreConfig_InitFromPreConfig( PyPreConfig *preconfig, const PyPreConfig *config2); extern PyObject* _PyPreConfig_AsDict(const PyPreConfig *preconfig); @@ -135,8 +137,6 @@ extern PyStatus _PyPreConfig_Write(const PyPreConfig *preconfig); /* --- PyConfig ---------------------------------------------- */ -#define _Py_CONFIG_VERSION 1 - typedef enum { /* Py_Initialize() API: backward compatibility with Python 3.6 and 3.7 */ _PyConfig_INIT_COMPAT = 1, @@ -144,7 +144,7 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; -PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); +PyAPI_FUNC(PyStatus) _PyConfig_InitCompatConfig(PyConfig *config); extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); diff --git a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst new file mode 100644 index 000000000000..b797933d05d6 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst @@ -0,0 +1,3 @@ +Add a new ``struct_size`` field to :c:type:`PyPreConfig` and :c:type:`PyConfig` +structures to allow to modify these structures in the future without breaking +the backward compatibility. diff --git a/Modules/main.c b/Modules/main.c index b8a1c9b79ce3..13936add4f3e 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,13 +53,20 @@ pymain_init(const _PyArgv *args) #endif PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _Py_PreInitializeFromPyArgv(&preconfig, args); if (_PyStatus_EXCEPTION(status)) { return status; } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (_PyStatus_EXCEPTION(status)) { goto done; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 74735f279c58..7c494c2786c0 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -78,6 +78,7 @@ main(int argc, char *argv[]) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 83c266b885af..a37594524f3d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -385,7 +385,12 @@ static int check_init_compat_config(int preinit) if (preinit) { PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -394,7 +399,13 @@ static int check_init_compat_config(int preinit) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + config_set_program_name(&config); init_from_config_clear(&config); @@ -470,7 +481,12 @@ static int test_init_from_config(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } putenv("PYTHONMALLOC=malloc_debug"); preconfig.allocator = PYMEM_ALLOCATOR_MALLOC; @@ -485,7 +501,12 @@ static int test_init_from_config(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config.install_signal_handlers = 0; /* FIXME: test use_environment */ @@ -617,6 +638,8 @@ static int check_init_parse_argv(int parse_argv) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -702,6 +725,8 @@ static int test_init_python_env(void) set_all_env_vars(); PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -755,6 +780,8 @@ static int test_init_isolated_flag(void) /* Test PyConfig.isolated=1 */ PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -779,7 +806,13 @@ static int test_preinit_isolated1(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 1; status = Py_PreInitialize(&preconfig); @@ -788,7 +821,12 @@ static int test_preinit_isolated1(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config_set_program_name(&config); set_all_env_vars(); init_from_config_clear(&config); @@ -805,7 +843,13 @@ static int test_preinit_isolated2(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 0; status = Py_PreInitialize(&preconfig); @@ -815,7 +859,11 @@ static int test_preinit_isolated2(void) /* Test PyConfig.isolated=1 */ PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } Py_IsolatedFlag = 0; config.isolated = 1; @@ -835,7 +883,12 @@ static int test_preinit_dont_parse_argv(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.isolated = 0; @@ -852,6 +905,7 @@ static int test_preinit_dont_parse_argv(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { @@ -877,6 +931,7 @@ static int test_preinit_parse_argv(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -934,7 +989,12 @@ static int check_preinit_isolated_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -947,6 +1007,8 @@ static int check_preinit_isolated_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); @@ -996,7 +1058,12 @@ static int check_init_python_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -1005,6 +1072,8 @@ static int check_init_python_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1035,7 +1104,13 @@ static int test_init_dont_configure_locale(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.configure_locale = 0; preconfig.coerce_c_locale = 1; preconfig.coerce_c_locale_warn = 1; @@ -1046,6 +1121,8 @@ static int test_init_dont_configure_locale(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1063,6 +1140,8 @@ static int test_init_dev_mode(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1286,6 +1365,8 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1334,6 +1415,8 @@ static int test_init_read_set(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1382,6 +1465,8 @@ static int test_init_sys_add(void) PySys_AddWarnOption(L"ignore:::sysadd_warnoption"); PyConfig config; + config.struct_size = sizeof(PyConfig); + PyStatus status; status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1450,7 +1535,12 @@ static int test_init_setpath_config(void) { PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } /* Explicitly preinitializes with Python preconfiguration to avoid Py_SetPath() implicit preinitialization with compat preconfiguration. */ @@ -1474,6 +1564,7 @@ static int test_init_setpath_config(void) putenv("TESTPATH="); PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1531,6 +1622,8 @@ static int test_init_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1546,6 +1639,7 @@ static int test_init_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1577,6 +1671,7 @@ static int test_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Python/frozenmain.c b/Python/frozenmain.c index c56938ab4899..76309e9e5da2 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -40,6 +40,7 @@ Py_FrozenMain(int argc, char **argv) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); diff --git a/Python/initconfig.c b/Python/initconfig.c index a527271452ea..dbc9de55a582 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -528,6 +528,18 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv) ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY()) + +static PyStatus +config_check_struct_size(const PyConfig *config) +{ + if (config->struct_size != sizeof(PyConfig)) { + return _PyStatus_ERR("unsupported PyConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + /* Free memory allocated in config, but don't clear all attributes */ void PyConfig_Clear(PyConfig *config) @@ -568,12 +580,19 @@ PyConfig_Clear(PyConfig *config) } -void +PyStatus _PyConfig_InitCompatConfig(PyConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->isolated = -1; config->use_environment = -1; @@ -603,13 +622,17 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + return _PyStatus_OK(); } -static void +static PyStatus config_init_defaults(PyConfig *config) { - _PyConfig_InitCompatConfig(config); + PyStatus status = _PyConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } config->isolated = 0; config->use_environment = 1; @@ -628,13 +651,18 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif + return _PyStatus_OK(); } PyStatus PyConfig_InitPythonConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->configure_c_stdio = 1; @@ -647,7 +675,11 @@ PyConfig_InitPythonConfig(PyConfig *config) PyStatus PyConfig_InitIsolatedConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->isolated = 1; @@ -742,6 +774,19 @@ PyStatus _PyConfig_Copy(PyConfig *config, const PyConfig *config2) { PyStatus status; + + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + + status = config_check_struct_size(config2); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + PyConfig_Clear(config); #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR @@ -2199,7 +2244,12 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline) } PyPreConfig preconfig; - _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } _PyPreConfig_GetConfig(&preconfig, config); @@ -2338,6 +2388,12 @@ PyConfig_Read(PyConfig *config) PyStatus status; PyWideStringList orig_argv = PyWideStringList_INIT; + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + status = _Py_PreInitializeFromConfig(config, NULL); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index d5b8b1acff8a..41ef7d28d066 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -434,7 +434,12 @@ pathconfig_global_read(_PyPathConfig *pathconfig) { PyStatus status; PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } /* Call _PyConfig_InitPathConfig() */ status = PyConfig_Read(&config); diff --git a/Python/preconfig.c b/Python/preconfig.c index 8be6533eace0..d18b01dc4586 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -269,12 +269,30 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig) /* --- PyPreConfig ----------------------------------------------- */ -void +static PyStatus +preconfig_check_struct_size(PyPreConfig *config) +{ + if (config->struct_size != sizeof(PyPreConfig)) { + return _PyStatus_ERR("unsupported PyPreConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + +PyStatus _PyPreConfig_InitCompatConfig(PyPreConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->parse_argv = 0; config->isolated = -1; @@ -295,13 +313,18 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = -1; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->isolated = 0; @@ -316,13 +339,18 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->configure_locale = 0; @@ -333,41 +361,55 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromPreConfig(PyPreConfig *config, const PyPreConfig *config2) { - PyPreConfig_InitPythonConfig(config); + PyStatus status = PyPreConfig_InitPythonConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + preconfig_copy(config, config2); + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config) { + PyStatus status; _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init; switch (config_init) { case _PyConfig_INIT_PYTHON: - PyPreConfig_InitPythonConfig(preconfig); + status = PyPreConfig_InitPythonConfig(preconfig); break; case _PyConfig_INIT_ISOLATED: - PyPreConfig_InitIsolatedConfig(preconfig); + status = PyPreConfig_InitIsolatedConfig(preconfig); break; case _PyConfig_INIT_COMPAT: default: - _PyPreConfig_InitCompatConfig(preconfig); + status = _PyPreConfig_InitCompatConfig(preconfig); } + + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyPreConfig_GetConfig(preconfig, config); + return _PyStatus_OK(); } static void preconfig_copy(PyPreConfig *config, const PyPreConfig *config2) { - assert(config2->_config_version == _Py_CONFIG_VERSION); + assert(config->struct_size == sizeof(PyPreConfig)); + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR COPY_ATTR(_config_init); @@ -787,6 +829,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) return status; } + status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + preconfig_get_global_vars(config); /* Copy LC_CTYPE locale, since it's modified later */ @@ -801,7 +849,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) /* Save the config to be able to restore it if encodings change */ PyPreConfig save_config; - _PyPreConfig_InitFromPreConfig(&save_config, config); + save_config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&save_config, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } /* Set LC_CTYPE to the user preferred locale */ if (config->configure_locale) { @@ -923,7 +976,12 @@ PyStatus _PyPreConfig_Write(const PyPreConfig *src_config) { PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (_PyRuntime.core_initialized) { /* bpo-34008: Calling this functions after Py_Initialize() ignores diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 42a062e10865..c154d060ad8d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -731,7 +731,12 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) runtime->preinitializing = 1; PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } status = _PyPreConfig_Read(&config, args); if (_PyStatus_EXCEPTION(status)) { @@ -790,7 +795,12 @@ _Py_PreInitializeFromConfig(const PyConfig *config, } PyPreConfig preconfig; - _PyPreConfig_InitFromConfig(&preconfig, config); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromConfig(&preconfig, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (!config->parse_argv) { return Py_PreInitialize(&preconfig); @@ -838,7 +848,12 @@ pyinit_core(_PyRuntimeState *runtime, } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } status = _PyConfig_Copy(&config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -1061,7 +1076,13 @@ Py_InitializeEx(int install_sigs) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } + config.install_signal_handlers = install_sigs; status = Py_InitializeFromConfig(&config); diff --git a/Python/pystate.c b/Python/pystate.c index f2924a8ff263..f0662fa70657 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -60,7 +60,12 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); - PyPreConfig_InitPythonConfig(&runtime->preconfig); + + runtime->preconfig.struct_size = sizeof(PyPreConfig); + PyStatus status = PyPreConfig_InitPythonConfig(&runtime->preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } runtime->gilstate.check_enabled = 1; @@ -205,6 +210,7 @@ PyInterpreterState_New(void) interp->id_refcount = -1; interp->check_interval = 100; + interp->config.struct_size = sizeof(PyConfig); PyStatus status = PyConfig_InitPythonConfig(&interp->config); if (_PyStatus_EXCEPTION(status)) { /* Don't report status to caller: PyConfig_InitPythonConfig() [View Less]

1 0

bpo-38304: Add PyConfig.struct_size (GH-16451)
by Victor Stinner Sept. 28, 2019

Sept. 28, 2019

https://github.com/python/cpython/commit/441b10cf2855955c86565f8d59e72c2efc… commit: 441b10cf2855955c86565f8d59e72c2efc0f0a57 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-09-28T04:28:35+02:00 summary: bpo-38304: Add PyConfig.struct_size (GH-16451) Add a new struct_size field to PyPreConfig and PyConfig structures to allow to modify these structures in the future without breaking the backward compatibility. * … [View More]Replace private _config_version field with public struct_size field in PyPreConfig and PyConfig. * Public PyPreConfig_InitIsolatedConfig() and PyPreConfig_InitPythonConfig() return type becomes PyStatus, instead of void. * Internal _PyConfig_InitCompatConfig(), _PyPreConfig_InitCompatConfig(), _PyPreConfig_InitFromConfig(), _PyPreConfig_InitFromPreConfig() return type becomes PyStatus, instead of void. * Remove _Py_CONFIG_VERSION * Update the Initialization Configuration documentation. files: A Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst M Doc/c-api/init_config.rst M Include/cpython/initconfig.h M Include/internal/pycore_initconfig.h M Modules/main.c M Programs/_freeze_importlib.c M Programs/_testembed.c M Python/frozenmain.c M Python/initconfig.c M Python/pathconfig.c M Python/preconfig.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0c3c725c841a..58e417441d13 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -194,18 +194,25 @@ PyPreConfig * Configure the LC_CTYPE locale * Set the UTF-8 mode + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyPreConfig)``. + Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Python Configuration <init-python-config>`. - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Isolated Configuration <init-isolated-conf>`. + The caller of these functions is responsible to handle exceptions (error or + exit) using :c:func:`PyStatus_Exception` and + :c:func:`Py_ExitStatusException`. + Structure fields: .. c:member:: int allocator @@ -267,6 +274,13 @@ PyPreConfig same way the regular Python parses command line arguments: see :ref:`Command Line Arguments <using-on-cmdline>`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyPreConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int use_environment See :c:member:`PyConfig.use_environment`. @@ -316,12 +330,18 @@ the preinitialization. Example using the preinitialization to enable the UTF-8 Mode:: + PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.utf8_mode = 1; - PyStatus status = Py_PreInitialize(&preconfig); + status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); } @@ -340,6 +360,9 @@ PyConfig Structure containing most parameters to configure Python. + The :c:member:`struct_size` field must be explicitly initialized to + ``sizeof(PyConfig)``. + Structure methods: .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config) @@ -656,6 +679,13 @@ PyConfig Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`. + .. c:member:: size_t struct_size + + Size of the structure in bytes: must be initialized to + ``sizeof(PyConfig)``. + + Field used for API and ABI compatibility. + .. c:member:: int tracemalloc If non-zero, call :func:`tracemalloc.start` at startup. @@ -718,6 +748,7 @@ Example setting the program name:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -750,6 +781,7 @@ configuration, and then override some parameters:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -835,8 +867,9 @@ Example of customized Python always running in isolated mode:: int main(int argc, char **argv) { - PyConfig config; PyStatus status; + PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1028,6 +1061,7 @@ phases:: { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 047f511f3104..d5effb8397fc 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -45,8 +45,10 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list, /* --- PyPreConfig ----------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyPreConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ /* Parse Py_PreInitializeFromBytesArgs() arguments? @@ -122,15 +124,17 @@ typedef struct { int allocator; } PyPreConfig; -PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config); -PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config); +PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); /* --- PyConfig ---------------------------------------------- */ typedef struct { - int _config_version; /* Internal configuration version, - used for ABI compatibility */ + /* Size of the structure in bytes: must be initialized to + sizeof(PyConfig). Field used for API and ABI compatibility. */ + size_t struct_size; + int _config_init; /* _PyConfigInitEnum value */ int isolated; /* Isolated mode? see PyPreConfig.isolated */ @@ -403,7 +407,6 @@ typedef struct { /* If equal to 0, stop Python initialization before the "main" phase */ int _init_main; - } PyConfig; PyAPI_FUNC(PyStatus) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index a7c1994eca16..dcdb61695cab 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -40,6 +40,8 @@ extern "C" { (err._type == _PyStatus_TYPE_EXIT) #define _PyStatus_EXCEPTION(err) \ (err._type != _PyStatus_TYPE_OK) +#define _PyStatus_UPDATE_FUNC(err) \ + do { err.func = _PyStatus_GET_FUNC(); } while (0) /* --- PyWideStringList ------------------------------------------------ */ @@ -118,11 +120,11 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline, /* --- PyPreConfig ----------------------------------------------- */ -PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); -extern void _PyPreConfig_InitFromConfig( +PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); +extern PyStatus _PyPreConfig_InitFromConfig( PyPreConfig *preconfig, const PyConfig *config); -extern void _PyPreConfig_InitFromPreConfig( +extern PyStatus _PyPreConfig_InitFromPreConfig( PyPreConfig *preconfig, const PyPreConfig *config2); extern PyObject* _PyPreConfig_AsDict(const PyPreConfig *preconfig); @@ -135,8 +137,6 @@ extern PyStatus _PyPreConfig_Write(const PyPreConfig *preconfig); /* --- PyConfig ---------------------------------------------- */ -#define _Py_CONFIG_VERSION 1 - typedef enum { /* Py_Initialize() API: backward compatibility with Python 3.6 and 3.7 */ _PyConfig_INIT_COMPAT = 1, @@ -144,7 +144,7 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; -PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); +PyAPI_FUNC(PyStatus) _PyConfig_InitCompatConfig(PyConfig *config); extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); diff --git a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst new file mode 100644 index 000000000000..b797933d05d6 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst @@ -0,0 +1,3 @@ +Add a new ``struct_size`` field to :c:type:`PyPreConfig` and :c:type:`PyConfig` +structures to allow to modify these structures in the future without breaking +the backward compatibility. diff --git a/Modules/main.c b/Modules/main.c index fdc7e0dabf17..ef6c66a8dbe5 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,13 +53,20 @@ pymain_init(const _PyArgv *args) #endif PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _Py_PreInitializeFromPyArgv(&preconfig, args); if (_PyStatus_EXCEPTION(status)) { return status; } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (_PyStatus_EXCEPTION(status)) { goto done; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 74735f279c58..7c494c2786c0 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -78,6 +78,7 @@ main(int argc, char *argv[]) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 83c266b885af..a37594524f3d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -385,7 +385,12 @@ static int check_init_compat_config(int preinit) if (preinit) { PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -394,7 +399,13 @@ static int check_init_compat_config(int preinit) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + config_set_program_name(&config); init_from_config_clear(&config); @@ -470,7 +481,12 @@ static int test_init_from_config(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } putenv("PYTHONMALLOC=malloc_debug"); preconfig.allocator = PYMEM_ALLOCATOR_MALLOC; @@ -485,7 +501,12 @@ static int test_init_from_config(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config.install_signal_handlers = 0; /* FIXME: test use_environment */ @@ -617,6 +638,8 @@ static int check_init_parse_argv(int parse_argv) PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -702,6 +725,8 @@ static int test_init_python_env(void) set_all_env_vars(); PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -755,6 +780,8 @@ static int test_init_isolated_flag(void) /* Test PyConfig.isolated=1 */ PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -779,7 +806,13 @@ static int test_preinit_isolated1(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 1; status = Py_PreInitialize(&preconfig); @@ -788,7 +821,12 @@ static int test_preinit_isolated1(void) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } config_set_program_name(&config); set_all_env_vars(); init_from_config_clear(&config); @@ -805,7 +843,13 @@ static int test_preinit_isolated2(void) PyStatus status; PyPreConfig preconfig; - _PyPreConfig_InitCompatConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitCompatConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.isolated = 0; status = Py_PreInitialize(&preconfig); @@ -815,7 +859,11 @@ static int test_preinit_isolated2(void) /* Test PyConfig.isolated=1 */ PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + status = _PyConfig_InitCompatConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } Py_IsolatedFlag = 0; config.isolated = 1; @@ -835,7 +883,12 @@ static int test_preinit_dont_parse_argv(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } preconfig.isolated = 0; @@ -852,6 +905,7 @@ static int test_preinit_dont_parse_argv(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { @@ -877,6 +931,7 @@ static int test_preinit_parse_argv(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -934,7 +989,12 @@ static int check_preinit_isolated_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitIsolatedConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -947,6 +1007,8 @@ static int check_preinit_isolated_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitIsolatedConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); @@ -996,7 +1058,12 @@ static int check_init_python_config(int preinit) if (preinit) { PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { @@ -1005,6 +1072,8 @@ static int check_init_python_config(int preinit) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1035,7 +1104,13 @@ static int test_init_dont_configure_locale(void) PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + preconfig.configure_locale = 0; preconfig.coerce_c_locale = 1; preconfig.coerce_c_locale_warn = 1; @@ -1046,6 +1121,8 @@ static int test_init_dont_configure_locale(void) } PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1063,6 +1140,8 @@ static int test_init_dev_mode(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1286,6 +1365,8 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1334,6 +1415,8 @@ static int test_init_read_set(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1382,6 +1465,8 @@ static int test_init_sys_add(void) PySys_AddWarnOption(L"ignore:::sysadd_warnoption"); PyConfig config; + config.struct_size = sizeof(PyConfig); + PyStatus status; status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1450,7 +1535,12 @@ static int test_init_setpath_config(void) { PyStatus status; PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = PyPreConfig_InitPythonConfig(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } /* Explicitly preinitializes with Python preconfiguration to avoid Py_SetPath() implicit preinitialization with compat preconfiguration. */ @@ -1474,6 +1564,7 @@ static int test_init_setpath_config(void) putenv("TESTPATH="); PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1531,6 +1622,8 @@ static int test_init_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -1546,6 +1639,7 @@ static int test_init_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { @@ -1577,6 +1671,7 @@ static int test_run_main(void) { PyStatus status; PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { diff --git a/Python/frozenmain.c b/Python/frozenmain.c index c56938ab4899..76309e9e5da2 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -40,6 +40,7 @@ Py_FrozenMain(int argc, char **argv) } PyConfig config; + config.struct_size = sizeof(PyConfig); status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); diff --git a/Python/initconfig.c b/Python/initconfig.c index 9f04e3d183ab..62c868d5cbc9 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -528,6 +528,18 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv) ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY()) + +static PyStatus +config_check_struct_size(const PyConfig *config) +{ + if (config->struct_size != sizeof(PyConfig)) { + return _PyStatus_ERR("unsupported PyConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + /* Free memory allocated in config, but don't clear all attributes */ void PyConfig_Clear(PyConfig *config) @@ -568,12 +580,19 @@ PyConfig_Clear(PyConfig *config) } -void +PyStatus _PyConfig_InitCompatConfig(PyConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->isolated = -1; config->use_environment = -1; @@ -603,13 +622,17 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + return _PyStatus_OK(); } -static void +static PyStatus config_init_defaults(PyConfig *config) { - _PyConfig_InitCompatConfig(config); + PyStatus status = _PyConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } config->isolated = 0; config->use_environment = 1; @@ -628,13 +651,18 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif + return _PyStatus_OK(); } PyStatus PyConfig_InitPythonConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->configure_c_stdio = 1; @@ -647,7 +675,11 @@ PyConfig_InitPythonConfig(PyConfig *config) PyStatus PyConfig_InitIsolatedConfig(PyConfig *config) { - config_init_defaults(config); + PyStatus status = config_init_defaults(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->isolated = 1; @@ -742,6 +774,19 @@ PyStatus _PyConfig_Copy(PyConfig *config, const PyConfig *config2) { PyStatus status; + + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + + status = config_check_struct_size(config2); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + PyConfig_Clear(config); #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR @@ -2204,7 +2249,12 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline) } PyPreConfig preconfig; - _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } _PyPreConfig_GetConfig(&preconfig, config); @@ -2385,6 +2435,12 @@ PyConfig_Read(PyConfig *config) PyStatus status; PyWideStringList orig_argv = PyWideStringList_INIT; + status = config_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + status = _Py_PreInitializeFromConfig(config, NULL); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 812614532800..6886ab7c42d6 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -434,7 +434,12 @@ pathconfig_global_read(_PyPathConfig *pathconfig) { PyStatus status; PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } /* Call _PyConfig_InitPathConfig() */ status = PyConfig_Read(&config); diff --git a/Python/preconfig.c b/Python/preconfig.c index 8be6533eace0..d18b01dc4586 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -269,12 +269,30 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig) /* --- PyPreConfig ----------------------------------------------- */ -void +static PyStatus +preconfig_check_struct_size(PyPreConfig *config) +{ + if (config->struct_size != sizeof(PyPreConfig)) { + return _PyStatus_ERR("unsupported PyPreConfig structure size " + "(Python version mismatch?)"); + } + return _PyStatus_OK(); +} + + +PyStatus _PyPreConfig_InitCompatConfig(PyPreConfig *config) { + size_t struct_size = config->struct_size; memset(config, 0, sizeof(*config)); + config->struct_size = struct_size; + + PyStatus status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } - config->_config_version = _Py_CONFIG_VERSION; config->_config_init = (int)_PyConfig_INIT_COMPAT; config->parse_argv = 0; config->isolated = -1; @@ -295,13 +313,18 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = -1; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_PYTHON; config->isolated = 0; @@ -316,13 +339,18 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *config) { - _PyPreConfig_InitCompatConfig(config); + PyStatus status = _PyPreConfig_InitCompatConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } config->_config_init = (int)_PyConfig_INIT_ISOLATED; config->configure_locale = 0; @@ -333,41 +361,55 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config) #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; #endif + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromPreConfig(PyPreConfig *config, const PyPreConfig *config2) { - PyPreConfig_InitPythonConfig(config); + PyStatus status = PyPreConfig_InitPythonConfig(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + preconfig_copy(config, config2); + return _PyStatus_OK(); } -void +PyStatus _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config) { + PyStatus status; _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init; switch (config_init) { case _PyConfig_INIT_PYTHON: - PyPreConfig_InitPythonConfig(preconfig); + status = PyPreConfig_InitPythonConfig(preconfig); break; case _PyConfig_INIT_ISOLATED: - PyPreConfig_InitIsolatedConfig(preconfig); + status = PyPreConfig_InitIsolatedConfig(preconfig); break; case _PyConfig_INIT_COMPAT: default: - _PyPreConfig_InitCompatConfig(preconfig); + status = _PyPreConfig_InitCompatConfig(preconfig); } + + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyPreConfig_GetConfig(preconfig, config); + return _PyStatus_OK(); } static void preconfig_copy(PyPreConfig *config, const PyPreConfig *config2) { - assert(config2->_config_version == _Py_CONFIG_VERSION); + assert(config->struct_size == sizeof(PyPreConfig)); + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR COPY_ATTR(_config_init); @@ -787,6 +829,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) return status; } + status = preconfig_check_struct_size(config); + if (_PyStatus_EXCEPTION(status)) { + _PyStatus_UPDATE_FUNC(status); + return status; + } + preconfig_get_global_vars(config); /* Copy LC_CTYPE locale, since it's modified later */ @@ -801,7 +849,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) /* Save the config to be able to restore it if encodings change */ PyPreConfig save_config; - _PyPreConfig_InitFromPreConfig(&save_config, config); + save_config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&save_config, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } /* Set LC_CTYPE to the user preferred locale */ if (config->configure_locale) { @@ -923,7 +976,12 @@ PyStatus _PyPreConfig_Write(const PyPreConfig *src_config) { PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (_PyRuntime.core_initialized) { /* bpo-34008: Calling this functions after Py_Initialize() ignores diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index eed583a4fdbc..ea0d7a5ee2b9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -735,7 +735,12 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) runtime->preinitializing = 1; PyPreConfig config; - _PyPreConfig_InitFromPreConfig(&config, src_config); + config.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } status = _PyPreConfig_Read(&config, args); if (_PyStatus_EXCEPTION(status)) { @@ -794,7 +799,12 @@ _Py_PreInitializeFromConfig(const PyConfig *config, } PyPreConfig preconfig; - _PyPreConfig_InitFromConfig(&preconfig, config); + preconfig.struct_size = sizeof(PyPreConfig); + + status = _PyPreConfig_InitFromConfig(&preconfig, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } if (!config->parse_argv) { return Py_PreInitialize(&preconfig); @@ -842,7 +852,12 @@ pyinit_core(_PyRuntimeState *runtime, } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } status = _PyConfig_Copy(&config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -1064,7 +1079,13 @@ Py_InitializeEx(int install_sigs) } PyConfig config; - _PyConfig_InitCompatConfig(&config); + config.struct_size = sizeof(PyConfig); + + status = _PyConfig_InitCompatConfig(&config); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } + config.install_signal_handlers = install_sigs; status = Py_InitializeFromConfig(&config); diff --git a/Python/pystate.c b/Python/pystate.c index 7dd8b7f866b5..0f0cb2299557 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -60,7 +60,12 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); - PyPreConfig_InitPythonConfig(&runtime->preconfig); + + runtime->preconfig.struct_size = sizeof(PyPreConfig); + PyStatus status = PyPreConfig_InitPythonConfig(&runtime->preconfig); + if (_PyStatus_EXCEPTION(status)) { + return status; + } runtime->gilstate.check_enabled = 1; @@ -204,6 +209,7 @@ PyInterpreterState_New(void) memset(interp, 0, sizeof(*interp)); interp->id_refcount = -1; + interp->config.struct_size = sizeof(PyConfig); PyStatus status = PyConfig_InitPythonConfig(&interp->config); if (_PyStatus_EXCEPTION(status)) { /* Don't report status to caller: PyConfig_InitPythonConfig() [View Less]

1 0