Mailman 3 October 2017 - Python-checkins

bpo-30406: Make async and await proper keywords (#1669)
by Yury Selivanov Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/ac317700ce7439e38a8b420218d9a5035b… commit: ac317700ce7439e38a8b420218d9a5035bba92ed branch: master author: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> committer: Yury Selivanov <yury(a)magic.io> date: 2017-10-05T23:24:46-04:00 summary: bpo-30406: Make async and await proper keywords (#1669) Per PEP 492, 'async' and 'await' should become proper keywords in 3.7. files: A Misc/NEWS.d/next/Core and Builtins/2017-07-20-22-03-44.bpo-30406._kr47t.rst M Doc/library/token.rst M Doc/tools/extensions/pyspecific.py M Grammar/Grammar M Include/graminit.h M Include/token.h M Lib/keyword.py M Lib/lib2to3/Grammar.txt M Lib/lib2to3/pgen2/token.py M Lib/lib2to3/pgen2/tokenize.py M Lib/lib2to3/tests/test_parser.py M Lib/pydoc.py M Lib/symbol.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_coroutines.py M Lib/test/test_parser.py M Lib/test/test_tokenize.py M Lib/tokenize.py M Parser/tokenizer.c M Parser/tokenizer.h M Python/ast.c M Python/graminit.c diff --git a/Doc/library/token.rst b/Doc/library/token.rst index b7ca9dbca72..373991027e4 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -98,8 +98,6 @@ The token constants are: RARROW ELLIPSIS OP - AWAIT - ASYNC ERRORTOKEN N_TOKENS NT_OFFSET @@ -129,9 +127,11 @@ the :mod:`tokenize` module. .. versionchanged:: 3.5 - Added :data:`AWAIT` and :data:`ASYNC` tokens. Starting with - Python 3.7, "async" and "await" will be tokenized as :data:`NAME` - tokens, and :data:`AWAIT` and :data:`ASYNC` will be removed. + Added :data:`AWAIT` and :data:`ASYNC` tokens. .. versionchanged:: 3.7 Added :data:`COMMENT`, :data:`NL` and :data:`ENCODING` tokens. + +.. versionchanged:: 3.7 + Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are + now tokenized as :data:`NAME` tokens. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index e8c88760a87..00acd4f55b8 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -272,9 +272,9 @@ def run(self): # Support for building "topic help" for pydoc pydoc_topic_labels = [ - 'assert', 'assignment', 'atom-identifiers', 'atom-literals', - 'attribute-access', 'attribute-references', 'augassign', 'binary', - 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', + 'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals', + 'attribute-access', 'attribute-references', 'augassign', 'await', + 'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', 'bltin-null-object', 'bltin-type-objects', 'booleans', 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', 'context-managers', 'continue', 'conversions', 'customization', 'debugger', diff --git a/Grammar/Grammar b/Grammar/Grammar index 90582434bee..7d3dd0b86dc 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -16,7 +16,7 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef +async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' @@ -68,7 +68,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -103,7 +103,7 @@ arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom_expr ['**' factor] -atom_expr: [AWAIT] atom trailer* +atom_expr: ['await'] atom trailer* atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | @@ -139,7 +139,8 @@ argument: ( test [comp_for] | '*' test ) comp_iter: comp_for | comp_if -comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] +sync_comp_for: 'for' exprlist 'in' or_test [comp_iter] +comp_for: ['async'] sync_comp_for comp_if: 'if' test_nocond [comp_iter] # not used in grammar, but may appear in "node" passed from Parser to Compiler diff --git a/Include/graminit.h b/Include/graminit.h index e9b4a938595..bdfe821ad71 100644 --- a/Include/graminit.h +++ b/Include/graminit.h @@ -81,8 +81,9 @@ #define arglist 334 #define argument 335 #define comp_iter 336 -#define comp_for 337 -#define comp_if 338 -#define encoding_decl 339 -#define yield_expr 340 -#define yield_arg 341 +#define sync_comp_for 337 +#define comp_for 338 +#define comp_if 339 +#define encoding_decl 340 +#define yield_expr 341 +#define yield_arg 342 diff --git a/Include/token.h b/Include/token.h index b28830b8b40..cd1cd00f09c 100644 --- a/Include/token.h +++ b/Include/token.h @@ -9,77 +9,75 @@ extern "C" { #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */ -#define ENDMARKER 0 -#define NAME 1 -#define NUMBER 2 -#define STRING 3 -#define NEWLINE 4 -#define INDENT 5 -#define DEDENT 6 -#define LPAR 7 -#define RPAR 8 -#define LSQB 9 -#define RSQB 10 -#define COLON 11 -#define COMMA 12 -#define SEMI 13 -#define PLUS 14 -#define MINUS 15 -#define STAR 16 -#define SLASH 17 -#define VBAR 18 -#define AMPER 19 -#define LESS 20 -#define GREATER 21 -#define EQUAL 22 -#define DOT 23 -#define PERCENT 24 -#define LBRACE 25 -#define RBRACE 26 -#define EQEQUAL 27 -#define NOTEQUAL 28 -#define LESSEQUAL 29 -#define GREATEREQUAL 30 -#define TILDE 31 -#define CIRCUMFLEX 32 -#define LEFTSHIFT 33 -#define RIGHTSHIFT 34 -#define DOUBLESTAR 35 -#define PLUSEQUAL 36 -#define MINEQUAL 37 -#define STAREQUAL 38 -#define SLASHEQUAL 39 -#define PERCENTEQUAL 40 -#define AMPEREQUAL 41 -#define VBAREQUAL 42 -#define CIRCUMFLEXEQUAL 43 -#define LEFTSHIFTEQUAL 44 -#define RIGHTSHIFTEQUAL 45 -#define DOUBLESTAREQUAL 46 -#define DOUBLESLASH 47 +#define ENDMARKER 0 +#define NAME 1 +#define NUMBER 2 +#define STRING 3 +#define NEWLINE 4 +#define INDENT 5 +#define DEDENT 6 +#define LPAR 7 +#define RPAR 8 +#define LSQB 9 +#define RSQB 10 +#define COLON 11 +#define COMMA 12 +#define SEMI 13 +#define PLUS 14 +#define MINUS 15 +#define STAR 16 +#define SLASH 17 +#define VBAR 18 +#define AMPER 19 +#define LESS 20 +#define GREATER 21 +#define EQUAL 22 +#define DOT 23 +#define PERCENT 24 +#define LBRACE 25 +#define RBRACE 26 +#define EQEQUAL 27 +#define NOTEQUAL 28 +#define LESSEQUAL 29 +#define GREATEREQUAL 30 +#define TILDE 31 +#define CIRCUMFLEX 32 +#define LEFTSHIFT 33 +#define RIGHTSHIFT 34 +#define DOUBLESTAR 35 +#define PLUSEQUAL 36 +#define MINEQUAL 37 +#define STAREQUAL 38 +#define SLASHEQUAL 39 +#define PERCENTEQUAL 40 +#define AMPEREQUAL 41 +#define VBAREQUAL 42 +#define CIRCUMFLEXEQUAL 43 +#define LEFTSHIFTEQUAL 44 +#define RIGHTSHIFTEQUAL 45 +#define DOUBLESTAREQUAL 46 +#define DOUBLESLASH 47 #define DOUBLESLASHEQUAL 48 #define AT 49 -#define ATEQUAL 50 +#define ATEQUAL 50 #define RARROW 51 #define ELLIPSIS 52 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */ -#define OP 53 -#define AWAIT 54 -#define ASYNC 55 -#define ERRORTOKEN 56 +#define OP 53 +#define ERRORTOKEN 54 /* These aren't used by the C tokenizer but are needed for tokenize.py */ -#define COMMENT 57 -#define NL 58 -#define ENCODING 59 -#define N_TOKENS 60 +#define COMMENT 55 +#define NL 56 +#define ENCODING 57 +#define N_TOKENS 58 /* Special definitions for cooperation with parser */ -#define NT_OFFSET 256 +#define NT_OFFSET 256 -#define ISTERMINAL(x) ((x) < NT_OFFSET) -#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) -#define ISEOF(x) ((x) == ENDMARKER) +#define ISTERMINAL(x) ((x) < NT_OFFSET) +#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) +#define ISEOF(x) ((x) == ENDMARKER) PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */ diff --git a/Lib/keyword.py b/Lib/keyword.py index 6e1e882a91e..431991dcf4a 100755 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -20,6 +20,8 @@ 'and', 'as', 'assert', + 'async', + 'await', 'break', 'class', 'continue', diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index ded032522ba..0bdfcafcf3c 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -34,7 +34,7 @@ eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef +async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* @@ -85,7 +85,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -124,7 +124,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)* arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -power: [AWAIT] atom trailer* ['**' factor] +power: ['await'] atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | @@ -161,7 +161,7 @@ argument: ( test [comp_for] | star_expr ) comp_iter: comp_for | comp_if -comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] +comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' old_test [comp_iter] # As noted above, testlist_safe extends the syntax allowed in list @@ -180,7 +180,7 @@ comp_if: 'if' old_test [comp_iter] # # See https://bugs.python.org/issue27494 old_comp_iter: old_comp_for | old_comp_if -old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter] +old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter] old_comp_if: 'if' old_test [old_comp_iter] testlist1: test (',' test)* diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 1a679554d2d..7599396611b 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -62,10 +62,8 @@ COMMENT = 53 NL = 54 RARROW = 55 -AWAIT = 56 -ASYNC = 57 -ERRORTOKEN = 58 -N_TOKENS = 59 +ERRORTOKEN = 56 +N_TOKENS = 57 NT_OFFSET = 256 #--end constants-- diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 45afc5f4e53..14560e4fddf 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -234,7 +234,7 @@ def compat(self, token, iterable): for tok in iterable: toknum, tokval = tok[:2] - if toknum in (NAME, NUMBER, ASYNC, AWAIT): + if toknum in (NAME, NUMBER): tokval += ' ' if toknum == INDENT: @@ -380,12 +380,6 @@ def generate_tokens(readline): contline = None indents = [0] - # 'stashed' and 'async_*' are used for async/await parsing - stashed = None - async_def = False - async_def_indent = 0 - async_def_nl = False - while 1: # loop over lines in stream try: line = readline() @@ -426,10 +420,6 @@ def generate_tokens(readline): pos = pos + 1 if pos == max: break - if stashed: - yield stashed - stashed = None - if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') @@ -453,18 +443,8 @@ def generate_tokens(readline): ("<tokenize>", lnum, pos, line)) indents = indents[:-1] - if async_def and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) - if async_def and async_def_nl and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) @@ -484,18 +464,10 @@ def generate_tokens(readline): newline = NEWLINE if parenlev > 0: newline = NL - elif async_def: - async_def_nl = True - if stashed: - yield stashed - stashed = None yield (newline, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") - if stashed: - yield stashed - stashed = None yield (COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] @@ -503,9 +475,6 @@ def generate_tokens(readline): if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] - if stashed: - yield stashed - stashed = None yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines @@ -523,63 +492,22 @@ def generate_tokens(readline): contline = line break else: # ordinary string - if stashed: - yield stashed - stashed = None yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name - if token in ('async', 'await'): - if async_def: - yield (ASYNC if token == 'async' else AWAIT, - token, spos, epos, line) - continue - - tok = (NAME, token, spos, epos, line) - if token == 'async' and not stashed: - stashed = tok - continue - - if token == 'def': - if (stashed - and stashed[0] == NAME - and stashed[1] == 'async'): - - async_def = True - async_def_indent = indents[-1] - - yield (ASYNC, stashed[1], - stashed[2], stashed[3], - stashed[4]) - stashed = None - - if stashed: - yield stashed - stashed = None - - yield tok + yield (NAME, token, spos, epos, line) elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: - if stashed: - yield stashed - stashed = None yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 - if stashed: - yield stashed - stashed = None yield (OP, token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1 - if stashed: - yield stashed - stashed = None - for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 2efcb80c2f9..dc94a69036a 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -167,34 +167,34 @@ def foo(): pass async def foo(): await x """) - self.invalid_syntax("await x") - self.invalid_syntax("""def foo(): - await x""") + self.validate("await x") + self.validate("""def foo(): + await x""") - self.invalid_syntax("""def foo(): + self.validate("""def foo(): def foo(): pass async def foo(): pass await x """) def test_async_var(self): - self.validate("""async = 1""") - self.validate("""await = 1""") - self.validate("""def async(): pass""") + self.invalid_syntax("""async = 1""") + self.invalid_syntax("""await = 1""") + self.invalid_syntax("""def async(): pass""") def test_async_with(self): self.validate("""async def foo(): async for a in b: pass""") - self.invalid_syntax("""def foo(): - async for a in b: pass""") + self.validate("""def foo(): + async for a in b: pass""") def test_async_for(self): self.validate("""async def foo(): async with a: pass""") - self.invalid_syntax("""def foo(): - async with a: pass""") + self.validate("""def foo(): + async with a: pass""") class TestRaiseChanges(GrammarTest): @@ -477,3 +477,7 @@ def diff(fn, result): os.remove("@") except OSError: pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 8dc3c0ace3c..01f7a32f454 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1703,7 +1703,7 @@ class Helper: # in pydoc_data/topics.py. # # CAUTION: if you change one of these dictionaries, be sure to adapt the - # list of needed labels in Doc/tools/pyspecific.py and + # list of needed labels in Doc/tools/extensions/pyspecific.py and # regenerate the pydoc_data/topics.py file by running # make pydoc-topics # in Doc/ and copying the output file into the Lib/ directory. @@ -1715,6 +1715,8 @@ class Helper: 'and': 'BOOLEAN', 'as': 'with', 'assert': ('assert', ''), + 'async': ('async', ''), + 'await': ('await', ''), 'break': ('break', 'while for'), 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), diff --git a/Lib/symbol.py b/Lib/symbol.py index d9f01e081a7..dc7dcba5e4d 100755 --- a/Lib/symbol.py +++ b/Lib/symbol.py @@ -91,11 +91,12 @@ arglist = 334 argument = 335 comp_iter = 336 -comp_for = 337 -comp_if = 338 -encoding_decl = 339 -yield_expr = 340 -yield_arg = 341 +sync_comp_for = 337 +comp_for = 338 +comp_if = 339 +encoding_decl = 340 +yield_expr = 341 +yield_arg = 342 #--end constants-- sym_name = {} diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 7ff56b560b6..e23963a6221 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -231,12 +231,6 @@ def test_ensure_future_neither(self): with self.assertRaises(TypeError): asyncio.ensure_future('ok') - def test_async_warning(self): - f = self.new_future(self.loop) - with self.assertWarnsRegex(DeprecationWarning, - 'function is deprecated, use ensure_'): - self.assertIs(f, asyncio.async(f)) - def test_get_stack(self): T = None diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 2b79a17ea70..ebd880bab0c 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -394,20 +394,14 @@ def async(): pass ] for code in samples: - with self.subTest(code=code), self.assertWarnsRegex( - DeprecationWarning, - "'await' will become reserved keywords"): + with self.subTest(code=code), self.assertRaises(SyntaxError): compile(code, "<test>", "exec") def test_badsyntax_3(self): - with self.assertRaises(DeprecationWarning): - with warnings.catch_warnings(): - warnings.simplefilter("error") - compile("async = 1", "<test>", "exec") - - def test_goodsyntax_1(self): - # Tests for issue 24619 + with self.assertRaises(SyntaxError): + compile("async = 1", "<test>", "exec") + def test_badsyntax_4(self): samples = [ '''def foo(await): async def foo(): pass @@ -454,14 +448,8 @@ def foo(): pass ] for code in samples: - with self.subTest(code=code): - loc = {} - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - exec(code, loc, loc) - - self.assertEqual(loc['foo'](10), 11) + with self.subTest(code=code), self.assertRaises(SyntaxError): + compile(code, "<test>", "exec") class TokenizerRegrTest(unittest.TestCase): diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 70cabb28598..647d391c798 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -679,16 +679,16 @@ def test_missing_import_source(self): def test_illegal_encoding(self): # Illegal encoding declaration tree = \ - (339, + (340, (257, (0, ''))) self.check_bad_tree(tree, "missed encoding") tree = \ - (339, + (340, (257, (0, '')), b'iso-8859-1') self.check_bad_tree(tree, "non-string encoding") tree = \ - (339, + (340, (257, (0, '')), '\udcff') with self.assertRaises(UnicodeEncodeError): diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 21eee6de2d1..3520a67bd42 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -759,7 +759,7 @@ def test_async(self): """) self.check_tokenize("async def foo(): pass", """\ - ASYNC 'async' (1, 0) (1, 5) + NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) @@ -776,7 +776,7 @@ def foo(await): await async += 1 ''', """\ - ASYNC 'async' (1, 0) (1, 5) + NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) @@ -787,12 +787,12 @@ def foo(await): NAME 'def' (2, 2) (2, 5) NAME 'foo' (2, 6) (2, 9) OP '(' (2, 9) (2, 10) - AWAIT 'await' (2, 10) (2, 15) + NAME 'await' (2, 10) (2, 15) OP ')' (2, 15) (2, 16) OP ':' (2, 16) (2, 17) NEWLINE '\\n' (2, 17) (2, 18) INDENT ' ' (3, 0) (3, 4) - AWAIT 'await' (3, 4) (3, 9) + NAME 'await' (3, 4) (3, 9) OP '=' (3, 10) (3, 11) NUMBER '1' (3, 12) (3, 13) NEWLINE '\\n' (3, 13) (3, 14) @@ -802,7 +802,7 @@ def foo(await): OP ':' (4, 6) (4, 7) NEWLINE '\\n' (4, 7) (4, 8) INDENT ' ' (5, 0) (5, 4) - AWAIT 'await' (5, 4) (5, 9) + NAME 'await' (5, 4) (5, 9) NEWLINE '\\n' (5, 9) (5, 10) DEDENT '' (6, 0) (6, 0) DEDENT '' (6, 0) (6, 0) @@ -815,7 +815,7 @@ def foo(await): self.check_tokenize('''\ async def foo(): async for i in 1: pass''', """\ - ASYNC 'async' (1, 0) (1, 5) + NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) @@ -823,7 +823,7 @@ def foo(await): OP ':' (1, 15) (1, 16) NEWLINE '\\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) - ASYNC 'async' (2, 2) (2, 7) + NAME 'async' (2, 2) (2, 7) NAME 'for' (2, 8) (2, 11) NAME 'i' (2, 12) (2, 13) NAME 'in' (2, 14) (2, 16) @@ -834,14 +834,14 @@ def foo(await): """) self.check_tokenize('''async def foo(async): await''', """\ - ASYNC 'async' (1, 0) (1, 5) + NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) - ASYNC 'async' (1, 14) (1, 19) + NAME 'async' (1, 14) (1, 19) OP ')' (1, 19) (1, 20) OP ':' (1, 20) (1, 21) - AWAIT 'await' (1, 22) (1, 27) + NAME 'await' (1, 22) (1, 27) """) self.check_tokenize('''\ @@ -866,7 +866,7 @@ def baz(): pass OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\\n' (3, 17) (3, 18) - ASYNC 'async' (4, 2) (4, 7) + NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) @@ -888,7 +888,7 @@ def baz(): pass async def bar(): pass await = 2''', """\ - ASYNC 'async' (1, 0) (1, 5) + NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'f' (1, 10) (1, 11) OP '(' (1, 11) (1, 12) @@ -904,7 +904,7 @@ def baz(): pass OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\\n' (3, 17) (3, 18) - ASYNC 'async' (4, 2) (4, 7) + NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) @@ -913,7 +913,7 @@ def baz(): pass NAME 'pass' (4, 19) (4, 23) NEWLINE '\\n' (4, 23) (4, 24) NL '\\n' (5, 0) (5, 1) - AWAIT 'await' (6, 2) (6, 7) + NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 5fa41526093..f5c6ac7f5e0 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -491,12 +491,6 @@ def _tokenize(readline, encoding): contline = None indents = [0] - # 'stashed' and 'async_*' are used for async/await parsing - stashed = None - async_def = False - async_def_indent = 0 - async_def_nl = False - if encoding is not None: if encoding == "utf-8-sig": # BOM will already have been stripped. @@ -571,18 +565,8 @@ def _tokenize(readline, encoding): ("<tokenize>", lnum, pos, line)) indents = indents[:-1] - if async_def and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) - if async_def and async_def_nl and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) @@ -601,21 +585,13 @@ def _tokenize(readline, encoding): (initial == '.' and token != '.' and token != '...')): yield TokenInfo(NUMBER, token, spos, epos, line) elif initial in '\r\n': - if stashed: - yield stashed - stashed = None if parenlev > 0: yield TokenInfo(NL, token, spos, epos, line) else: yield TokenInfo(NEWLINE, token, spos, epos, line) - if async_def: - async_def_nl = True elif initial == '#': assert not token.endswith("\n") - if stashed: - yield stashed - stashed = None yield TokenInfo(COMMENT, token, spos, epos, line) elif token in triple_quoted: @@ -662,36 +638,7 @@ def _tokenize(readline, encoding): yield TokenInfo(STRING, token, spos, epos, line) elif initial.isidentifier(): # ordinary name - if token in ('async', 'await'): - if async_def: - yield TokenInfo( - ASYNC if token == 'async' else AWAIT, - token, spos, epos, line) - continue - - tok = TokenInfo(NAME, token, spos, epos, line) - if token == 'async' and not stashed: - stashed = tok - continue - - if token == 'def': - if (stashed - and stashed.type == NAME - and stashed.string == 'async'): - - async_def = True - async_def_indent = indents[-1] - - yield TokenInfo(ASYNC, stashed.string, - stashed.start, stashed.end, - stashed.line) - stashed = None - - if stashed: - yield stashed - stashed = None - - yield tok + yield TokenInfo(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt continued = 1 else: @@ -699,19 +646,12 @@ def _tokenize(readline, encoding): parenlev += 1 elif initial in ')]}': parenlev -= 1 - if stashed: - yield stashed - stashed = None yield TokenInfo(OP, token, spos, epos, line) else: yield TokenInfo(ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos += 1 - if stashed: - yield stashed - stashed = None - for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-07-20-22-03-44.bpo-30406._kr47t.rst b/Misc/NEWS.d/next/Core and Builtins/2017-07-20-22-03-44.bpo-30406._kr47t.rst new file mode 100644 index 00000000000..caf56f03783 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-07-20-22-03-44.bpo-30406._kr47t.rst @@ -0,0 +1 @@ +Make ``async`` and ``await`` proper keywords, as specified in PEP 492. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 51f98e9b2e9..28254e10331 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -103,8 +103,6 @@ const char *_PyParser_TokenNames[] = { "ELLIPSIS", /* This table must match the #defines in token.h! */ "OP", - "AWAIT", - "ASYNC", "<ERRORTOKEN>", "COMMENT", "NL", @@ -151,10 +149,6 @@ tok_new(void) tok->decoding_buffer = NULL; #endif - tok->async_def = 0; - tok->async_def_indent = 0; - tok->async_def_nl = 0; - return tok; } @@ -1471,21 +1465,6 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) } } - if (tok->async_def - && !blankline - && tok->level == 0 - /* There was a NEWLINE after ASYNC DEF, - so we're past the signature. */ - && tok->async_def_nl - /* Current indentation level is less than where - the async function was defined */ - && tok->async_def_indent >= tok->indent) - { - tok->async_def = 0; - tok->async_def_indent = 0; - tok->async_def_nl = 0; - } - again: tok->start = NULL; /* Skip spaces */ @@ -1550,43 +1529,6 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) *p_start = tok->start; *p_end = tok->cur; - /* async/await parsing block. */ - if (tok->cur - tok->start == 5) { - /* Current token length is 5. */ - if (tok->async_def) { - /* We're inside an 'async def' function. */ - if (memcmp(tok->start, "async", 5) == 0) { - return ASYNC; - } - if (memcmp(tok->start, "await", 5) == 0) { - return AWAIT; - } - } - else if (memcmp(tok->start, "async", 5) == 0) { - /* The current token is 'async'. - Look ahead one token.*/ - - struct tok_state ahead_tok; - char *ahead_tok_start = NULL, *ahead_tok_end = NULL; - int ahead_tok_kind; - - memcpy(&ahead_tok, tok, sizeof(ahead_tok)); - ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start, - &ahead_tok_end); - - if (ahead_tok_kind == NAME - && ahead_tok.cur - ahead_tok.start == 3 - && memcmp(ahead_tok.start, "def", 3) == 0) - { - /* The next token is going to be 'def', so instead of - returning 'async' NAME token, we return ASYNC. */ - tok->async_def_indent = tok->indent; - tok->async_def = 1; - return ASYNC; - } - } - } - return NAME; } @@ -1599,11 +1541,6 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) *p_start = tok->start; *p_end = tok->cur - 1; /* Leave '\n' out of the string */ tok->cont_line = 0; - if (tok->async_def) { - /* We're somewhere inside an 'async def' function, and - we've encountered a NEWLINE after its signature. */ - tok->async_def_nl = 1; - } return NEWLINE; } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0ad3551c894..ad8b1c80171 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -65,13 +65,6 @@ struct tok_state { const char* enc; /* Encoding for the current str. */ const char* str; const char* input; /* Tokenizer's newline translated copy of the string. */ - - /* async/await related fields; can be removed in 3.7 when async and await - become normal keywords. */ - int async_def; /* =1 if tokens are inside an 'async def' body. */ - int async_def_indent; /* Indentation level of the outermost 'async def'. */ - int async_def_nl; /* =1 if the outermost 'async def' had at least one - NEWLINE token after it. */ }; extern struct tok_state *PyTokenizer_FromString(const char *, int); diff --git a/Python/ast.c b/Python/ast.c index 33356da0756..6989965efab 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -949,28 +949,6 @@ forbidden_name(struct compiling *c, identifier name, const node *n, ast_error(c, n, "assignment to keyword"); return 1; } - if (_PyUnicode_EqualToASCIIString(name, "async") || - _PyUnicode_EqualToASCIIString(name, "await")) - { - PyObject *message = PyUnicode_FromString( - "'async' and 'await' will become reserved keywords" - " in Python 3.7"); - int ret; - if (message == NULL) { - return 1; - } - ret = PyErr_WarnExplicitObject( - PyExc_DeprecationWarning, - message, - c->c_filename, - LINENO(n), - NULL, - NULL); - Py_DECREF(message); - if (ret < 0) { - return 1; - } - } if (full_checks) { const char * const *p; for (p = FORBIDDEN; *p; p++) { @@ -1642,9 +1620,10 @@ ast_for_funcdef_impl(struct compiling *c, const node *n, static stmt_ty ast_for_async_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) { - /* async_funcdef: ASYNC funcdef */ + /* async_funcdef: 'async' funcdef */ REQ(n, async_funcdef); - REQ(CHILD(n, 0), ASYNC); + REQ(CHILD(n, 0), NAME); + assert(strcmp(STR(CHILD(n, 0)), "async") == 0); REQ(CHILD(n, 1), funcdef); return ast_for_funcdef_impl(c, CHILD(n, 1), decorator_seq, @@ -1663,9 +1642,10 @@ ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) static stmt_ty ast_for_async_stmt(struct compiling *c, const node *n) { - /* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */ + /* async_stmt: 'async' (funcdef | with_stmt | for_stmt) */ REQ(n, async_stmt); - REQ(CHILD(n, 0), ASYNC); + REQ(CHILD(n, 0), NAME); + assert(strcmp(STR(CHILD(n, 0)), "async") == 0); switch (TYPE(CHILD(n, 1))) { case funcdef: @@ -1778,17 +1758,23 @@ static int count_comp_fors(struct compiling *c, const node *n) { int n_fors = 0; - int is_async; count_comp_for: - is_async = 0; n_fors++; REQ(n, comp_for); - if (TYPE(CHILD(n, 0)) == ASYNC) { - is_async = 1; + if (NCH(n) == 2) { + REQ(CHILD(n, 0), NAME); + assert(strcmp(STR(CHILD(n, 0)), "async") == 0); + n = CHILD(n, 1); } - if (NCH(n) == (5 + is_async)) { - n = CHILD(n, 4 + is_async); + else if (NCH(n) == 1) { + n = CHILD(n, 0); + } + else { + goto error; + } + if (NCH(n) == (5)) { + n = CHILD(n, 4); } else { return n_fors; @@ -1807,6 +1793,7 @@ count_comp_fors(struct compiling *c, const node *n) return n_fors; } + error: /* Should never be reached */ PyErr_SetString(PyExc_SystemError, "logic error in count_comp_fors"); @@ -1855,19 +1842,27 @@ ast_for_comprehension(struct compiling *c, const node *n) asdl_seq *t; expr_ty expression, first; node *for_ch; + node *sync_n; int is_async = 0; REQ(n, comp_for); - if (TYPE(CHILD(n, 0)) == ASYNC) { + if (NCH(n) == 2) { is_async = 1; + REQ(CHILD(n, 0), NAME); + assert(strcmp(STR(CHILD(n, 0)), "async") == 0); + sync_n = CHILD(n, 1); + } + else { + sync_n = CHILD(n, 0); } + REQ(sync_n, sync_comp_for); - for_ch = CHILD(n, 1 + is_async); + for_ch = CHILD(sync_n, 1); t = ast_for_exprlist(c, for_ch, Store); if (!t) return NULL; - expression = ast_for_expr(c, CHILD(n, 3 + is_async)); + expression = ast_for_expr(c, CHILD(sync_n, 3)); if (!expression) return NULL; @@ -1884,11 +1879,11 @@ ast_for_comprehension(struct compiling *c, const node *n) if (!comp) return NULL; - if (NCH(n) == (5 + is_async)) { + if (NCH(sync_n) == 5) { int j, n_ifs; asdl_seq *ifs; - n = CHILD(n, 4 + is_async); + n = CHILD(sync_n, 4); n_ifs = count_comp_ifs(c, n); if (n_ifs == -1) return NULL; @@ -2470,7 +2465,7 @@ ast_for_atom_expr(struct compiling *c, const node *n) REQ(n, atom_expr); nch = NCH(n); - if (TYPE(CHILD(n, 0)) == AWAIT) { + if (TYPE(CHILD(n, 0)) == NAME && strcmp(STR(CHILD(n, 0)), "await") == 0) { start = 1; assert(nch > 1); } @@ -2497,7 +2492,7 @@ ast_for_atom_expr(struct compiling *c, const node *n) } if (start) { - /* there was an AWAIT */ + /* there was an 'await' */ return Await(e, LINENO(n), n->n_col_offset, c->c_arena); } else { @@ -2562,7 +2557,7 @@ ast_for_expr(struct compiling *c, const node *n) term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom_expr ['**' factor] - atom_expr: [AWAIT] atom trailer* + atom_expr: ['await'] atom trailer* yield_expr: 'yield' [yield_arg] */ diff --git a/Python/graminit.c b/Python/graminit.c index f2584e0a2ad..8e89ccea3ba 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -1812,272 +1812,284 @@ static state states_80[2] = { {2, arcs_80_0}, {1, arcs_80_1}, }; -static arc arcs_81_0[2] = { - {21, 1}, - {101, 2}, +static arc arcs_81_0[1] = { + {101, 1}, }; static arc arcs_81_1[1] = { - {101, 2}, + {66, 2}, }; static arc arcs_81_2[1] = { - {66, 3}, + {102, 3}, }; static arc arcs_81_3[1] = { - {102, 4}, + {112, 4}, }; -static arc arcs_81_4[1] = { - {112, 5}, +static arc arcs_81_4[2] = { + {171, 5}, + {0, 4}, }; -static arc arcs_81_5[2] = { - {171, 6}, +static arc arcs_81_5[1] = { {0, 5}, }; -static arc arcs_81_6[1] = { - {0, 6}, -}; -static state states_81[7] = { - {2, arcs_81_0}, +static state states_81[6] = { + {1, arcs_81_0}, {1, arcs_81_1}, {1, arcs_81_2}, {1, arcs_81_3}, - {1, arcs_81_4}, - {2, arcs_81_5}, - {1, arcs_81_6}, + {2, arcs_81_4}, + {1, arcs_81_5}, }; -static arc arcs_82_0[1] = { - {97, 1}, +static arc arcs_82_0[2] = { + {21, 1}, + {173, 2}, }; static arc arcs_82_1[1] = { - {114, 2}, + {173, 2}, }; -static arc arcs_82_2[2] = { - {171, 3}, +static arc arcs_82_2[1] = { {0, 2}, }; -static arc arcs_82_3[1] = { - {0, 3}, -}; -static state states_82[4] = { - {1, arcs_82_0}, +static state states_82[3] = { + {2, arcs_82_0}, {1, arcs_82_1}, - {2, arcs_82_2}, - {1, arcs_82_3}, + {1, arcs_82_2}, }; static arc arcs_83_0[1] = { - {23, 1}, + {97, 1}, }; static arc arcs_83_1[1] = { - {0, 1}, + {114, 2}, +}; +static arc arcs_83_2[2] = { + {171, 3}, + {0, 2}, +}; +static arc arcs_83_3[1] = { + {0, 3}, }; -static state states_83[2] = { +static state states_83[4] = { {1, arcs_83_0}, {1, arcs_83_1}, + {2, arcs_83_2}, + {1, arcs_83_3}, }; static arc arcs_84_0[1] = { - {174, 1}, + {23, 1}, }; -static arc arcs_84_1[2] = { - {175, 2}, +static arc arcs_84_1[1] = { {0, 1}, }; -static arc arcs_84_2[1] = { +static state states_84[2] = { + {1, arcs_84_0}, + {1, arcs_84_1}, +}; +static arc arcs_85_0[1] = { + {175, 1}, +}; +static arc arcs_85_1[2] = { + {176, 2}, + {0, 1}, +}; +static arc arcs_85_2[1] = { {0, 2}, }; -static state states_84[3] = { - {1, arcs_84_0}, - {2, arcs_84_1}, - {1, arcs_84_2}, +static state states_85[3] = { + {1, arcs_85_0}, + {2, arcs_85_1}, + {1, arcs_85_2}, }; -static arc arcs_85_0[2] = { +static arc arcs_86_0[2] = { {77, 1}, {9, 2}, }; -static arc arcs_85_1[1] = { +static arc arcs_86_1[1] = { {26, 2}, }; -static arc arcs_85_2[1] = { +static arc arcs_86_2[1] = { {0, 2}, }; -static state states_85[3] = { - {2, arcs_85_0}, - {1, arcs_85_1}, - {1, arcs_85_2}, +static state states_86[3] = { + {2, arcs_86_0}, + {1, arcs_86_1}, + {1, arcs_86_2}, }; -static dfa dfas[86] = { +static dfa dfas[87] = { {256, "single_input", 0, 3, states_0, - "\004\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\102"}, + "\004\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\202\000"}, {257, "file_input", 0, 2, states_1, - "\204\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\102"}, + "\204\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\202\000"}, {258, "eval_input", 0, 3, states_2, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {259, "decorator", 0, 7, states_3, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {261, "decorated", 0, 3, states_5, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {262, "async_funcdef", 0, 3, states_6, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {263, "funcdef", 0, 8, states_7, - "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "parameters", 0, 4, states_8, - "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {265, "typedargslist", 0, 19, states_9, - "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {266, "tfpdef", 0, 4, states_10, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {267, "varargslist", 0, 19, states_11, - "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {268, "vfpdef", 0, 2, states_12, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {269, "stmt", 0, 2, states_13, - "\000\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\102"}, + "\000\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\202\000"}, {270, "simple_stmt", 0, 4, states_14, - "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\100"}, + "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\200\000"}, {271, "small_stmt", 0, 2, states_15, - "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\100"}, + "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\200\000"}, {272, "expr_stmt", 0, 6, states_16, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {273, "annassign", 0, 5, states_17, - "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {274, "testlist_star_expr", 0, 3, states_18, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {275, "augassign", 0, 2, states_19, - "\000\000\000\000\000\000\360\377\001\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\360\377\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {276, "del_stmt", 0, 3, states_20, - "\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {277, "pass_stmt", 0, 2, states_21, - "\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {278, "flow_stmt", 0, 2, states_22, - "\000\000\000\000\000\000\000\000\000\036\000\000\000\000\000\000\000\000\000\000\000\100"}, + "\000\000\000\000\000\000\000\000\000\036\000\000\000\000\000\000\000\000\000\000\000\200\000"}, {279, "break_stmt", 0, 2, states_23, - "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {280, "continue_stmt", 0, 2, states_24, - "\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {281, "return_stmt", 0, 3, states_25, - "\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {282, "yield_stmt", 0, 2, states_26, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\100"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"}, {283, "raise_stmt", 0, 5, states_27, - "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {284, "import_stmt", 0, 2, states_28, - "\000\000\000\000\000\000\000\000\000\040\001\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\040\001\000\000\000\000\000\000\000\000\000\000\000\000"}, {285, "import_name", 0, 3, states_29, - "\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000"}, {286, "import_from", 0, 8, states_30, - "\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {287, "import_as_name", 0, 4, states_31, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {288, "dotted_as_name", 0, 4, states_32, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {289, "import_as_names", 0, 3, states_33, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {290, "dotted_as_names", 0, 2, states_34, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {291, "dotted_name", 0, 2, states_35, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {292, "global_stmt", 0, 3, states_36, - "\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"}, {293, "nonlocal_stmt", 0, 3, states_37, - "\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000"}, {294, "assert_stmt", 0, 5, states_38, - "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000"}, {295, "compound_stmt", 0, 2, states_39, - "\000\010\140\000\000\000\000\000\000\000\000\000\262\004\000\000\000\000\000\000\000\002"}, + "\000\010\140\000\000\000\000\000\000\000\000\000\262\004\000\000\000\000\000\000\000\002\000"}, {296, "async_stmt", 0, 3, states_40, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {297, "if_stmt", 0, 8, states_41, - "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, {298, "while_stmt", 0, 8, states_42, - "\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, {299, "for_stmt", 0, 10, states_43, - "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, {300, "try_stmt", 0, 13, states_44, - "\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"}, {301, "with_stmt", 0, 5, states_45, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, {302, "with_item", 0, 4, states_46, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {303, "except_clause", 0, 5, states_47, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, {304, "suite", 0, 5, states_48, - "\004\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\100"}, + "\004\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\200\000"}, {305, "test", 0, 6, states_49, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {306, "test_nocond", 0, 2, states_50, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {307, "lambdef", 0, 5, states_51, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000"}, {308, "lambdef_nocond", 0, 5, states_52, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000"}, {309, "or_test", 0, 2, states_53, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000\000"}, {310, "and_test", 0, 2, states_54, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000\000"}, {311, "not_test", 0, 3, states_55, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000\000"}, {312, "comparison", 0, 2, states_56, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {313, "comp_op", 0, 4, states_57, - "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\362\017\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\362\017\000\000\000\000\000\000"}, {314, "star_expr", 0, 3, states_58, - "\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {315, "expr", 0, 2, states_59, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {316, "xor_expr", 0, 2, states_60, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {317, "and_expr", 0, 2, states_61, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {318, "shift_expr", 0, 2, states_62, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {319, "arith_expr", 0, 2, states_63, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {320, "term", 0, 2, states_64, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {321, "factor", 0, 3, states_65, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {322, "power", 0, 4, states_66, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\200\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\200\050\037\000\000"}, {323, "atom_expr", 0, 3, states_67, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\200\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\200\050\037\000\000"}, {324, "atom", 0, 9, states_68, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\050\037\000\000"}, {325, "testlist_comp", 0, 5, states_69, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {326, "trailer", 0, 7, states_70, - "\000\040\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\010\000\000"}, + "\000\040\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\010\000\000\000"}, {327, "subscriptlist", 0, 3, states_71, - "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {328, "subscript", 0, 5, states_72, - "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {329, "sliceop", 0, 3, states_73, - "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {330, "exprlist", 0, 3, states_74, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000"}, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, {331, "testlist", 0, 3, states_75, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {332, "dictorsetmaker", 0, 14, states_76, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {333, "classdef", 0, 8, states_77, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000"}, {334, "arglist", 0, 3, states_78, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {335, "argument", 0, 4, states_79, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000"}, + "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, {336, "comp_iter", 0, 2, states_80, - "\000\000\040\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000\000"}, - {337, "comp_for", 0, 7, states_81, - "\000\000\040\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, - {338, "comp_if", 0, 4, states_82, - "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"}, - {339, "encoding_decl", 0, 2, states_83, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {340, "yield_expr", 0, 3, states_84, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\100"}, - {341, "yield_arg", 0, 3, states_85, - "\000\040\200\000\000\000\000\000\000\040\010\000\000\000\020\002\000\300\220\050\037\000"}, -}; -static label labels[176] = { + "\000\000\040\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000\000\000"}, + {337, "sync_comp_for", 0, 6, states_81, + "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + {338, "comp_for", 0, 3, states_82, + "\000\000\040\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + {339, "comp_if", 0, 4, states_83, + "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, + {340, "encoding_decl", 0, 2, states_84, + "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {341, "yield_expr", 0, 3, states_85, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"}, + {342, "yield_arg", 0, 3, states_86, + "\000\040\200\000\000\000\000\000\000\040\010\000\000\000\020\002\000\300\220\050\037\000\000"}, +}; +static label labels[177] = { {0, "EMPTY"}, {256, 0}, {4, 0}, @@ -2099,7 +2111,7 @@ static label labels[176] = { {333, 0}, {263, 0}, {262, 0}, - {55, 0}, + {1, "async"}, {1, "def"}, {1, 0}, {264, 0}, @@ -2128,7 +2140,7 @@ static label labels[176] = { {274, 0}, {273, 0}, {275, 0}, - {340, 0}, + {341, 0}, {314, 0}, {36, 0}, {37, 0}, @@ -2229,7 +2241,7 @@ static label labels[176] = { {31, 0}, {322, 0}, {323, 0}, - {54, 0}, + {1, "await"}, {324, 0}, {326, 0}, {325, 0}, @@ -2243,21 +2255,22 @@ static label labels[176] = { {1, "None"}, {1, "True"}, {1, "False"}, - {337, 0}, + {338, 0}, {327, 0}, {328, 0}, {329, 0}, {1, "class"}, {335, 0}, {336, 0}, - {338, 0}, {339, 0}, + {337, 0}, + {340, 0}, {1, "yield"}, - {341, 0}, + {342, 0}, }; grammar _PyParser_Grammar = { - 86, + 87, dfas, - {176, labels}, + {177, labels}, 256 };

1 0

[3.6] bpo-31523: Reliability improvements to the Windows build files (GH-3900) (#3902)
by Steve Dower Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/051295a8c57cc649fa5eaa43526143984a… commit: 051295a8c57cc649fa5eaa43526143984a147411 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Steve Dower <steve.dower(a)microsoft.com> date: 2017-10-05T14:09:30-07:00 summary: [3.6] bpo-31523: Reliability improvements to the Windows build files (GH-3900) (#3902) (cherry picked from commit 2084b30e540d88b9fc752c5bdcc2f24334af4f2b) files: M PCbuild/pyproject.props M Tools/msi/msi.props M Tools/msi/purge.py M Tools/msi/uploadrelease.bat M Tools/msi/uploadrelease.proj diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 3d46a0fcba8..5a2da998c37 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -6,7 +6,7 @@ <OutDir>$(BuildPath)</OutDir> <OutDir Condition="!HasTrailingSlash($(OutDir))">$(OutDir)\</OutDir> <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir> - <IntDir>$(Py_IntDir)\$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir> + <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir> <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName> <TargetName>$(TargetName)$(PyDebugExt)</TargetName> <GenerateManifest>false</GenerateManifest> diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index 60abba1f7be..61095991088 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -48,7 +48,7 @@ </ItemGroup> <PropertyGroup> - <IntermediateOutputPath>$(Py_IntDir)\$(Configuration)_$(Platform)_Setup\$(OutputName)</IntermediateOutputPath> + <IntermediateOutputPath>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\msi_$(OutputName)</IntermediateOutputPath> <IntermediateOutputPath Condition="'$(OutputSuffix)' != ''">$(IntermediateOutputPath)_$(OutputSuffix)</IntermediateOutputPath> <OutputPath Condition="'$(OutputPath)' == ''">$(BuildPath)</OutputPath> <OutputPath Condition="!HasTrailingSlash($(OutputPath))">$(OutputPath)\</OutputPath> diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py index 76dbc49ae82..24fdae24388 100644 --- a/Tools/msi/purge.py +++ b/Tools/msi/purge.py @@ -25,7 +25,7 @@ sys.exit(1) URL = "https://www.python.org/ftp/python/{}/".format(m.group(1)) - +REL = m.group(2) or '' FILES = [ "core.msi", @@ -64,7 +64,7 @@ "python-{}-webinstall.exe".format(m.group(0)), "python-{}-amd64.exe".format(m.group(0)), "python-{}-amd64-webinstall.exe".format(m.group(0)), -] + ["win32{}/{}".format(m.group(2), f) for f in FILES] + ["amd64{}/{}".format(m.group(2), f) for f in FILES] +] + ["win32{}/{}".format(REL, f) for f in FILES] + ["amd64{}/{}".format(REL, f) for f in FILES] print('Purged:') for n in PATHS: diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 6441db9ee6f..d9dcc7a09e0 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -8,8 +8,9 @@ set HOST= set USER= set TARGET= set DRYRUN=false +set NOUPLOAD= set NOGPG= -set PURGE_OPTION=/p:Purge=true +set NOPURGE= set NOTEST= :CheckOpts @@ -21,8 +22,9 @@ if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts +if "%1" EQU "--skip-upload" (set NOUPLOAD=true) && shift && goto CheckOpts if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts -if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && goto CheckOpts +if "%1" EQU "--skip-purge" (set NOPURGE=true) && shift && goto CheckOpts if "%1" EQU "--skip-test" (set NOTEST=true) && shift && goto CheckOpts if "%1" EQU "-T" (set NOTEST=true) && shift && goto CheckOpts if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 @@ -52,17 +54,43 @@ if defined NOGPG ( call "%PCBUILD%find_msbuild.bat" %MSBUILD% if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) pushd "%D%" -%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% -%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +if not defined NOUPLOAD ( + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 + if errorlevel 1 goto :failed + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false + if errorlevel 1 goto :failed +) +if not defined NOPURGE ( + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge +) if not defined NOTEST ( - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 + call :test x86 + if errorlevel 1 goto :failed + call :test x64 + if errorlevel 1 goto :failed ) %MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 +if errorlevel 1 goto :failed %MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false +if errorlevel 1 goto :failed + popd exit /B 0 +:test +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1 +if errorlevel 1 ( + echo Test failed - purging and retrying + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge + if errorlevel 1 exit /B + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1 +) +exit /B + +:failed +popd +exit /B + :Help echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] echo. diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 75840f2f851..881b825e107 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -10,7 +10,6 @@ <IncludeDoc Condition="'$(IncludeDoc)' == ''">true</IncludeDoc> <BuildForRelease Condition="'$(BuildForRelease)' == ''">true</BuildForRelease> <DryRun Condition="'$(DryRun)' == ''">false</DryRun> - <Purge Condition="'$(Purge)' == ''">false</Purge> </PropertyGroup> <Import Project="msi.props" /> @@ -43,7 +42,8 @@ <Target Name="_RunGpg" Condition="'$(GPG)' != ''" Inputs="@(File)" Outputs="$(IntermediateOutputPath)\gpg\%(FileName)%(Extension).asc"> <MakeDir Directories="$(IntermediateOutputPath)gpg" /> <Delete Files="$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc" Condition="Exists('$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc')" /> - <Exec Command=""$(GPG)" -ba -o "$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc" "%(File.FullPath)"" /> + <Exec Command=""$(GPG)" -ba -o "$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc" "%(File.FullPath)"" + IgnoreExitCode="false" /> <ItemGroup> <File Include="$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc"> <CopyTo>%(File.CopyTo)</CopyTo> @@ -52,23 +52,15 @@ </Target> <Target Name="_Upload" Condition="!$(DryRun)"> - <Exec Command=""$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo) -"$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo) -"$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/* -" /> + <Exec Command=""$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo)" /> + <Exec Command=""$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo)" /> + <Exec Command=""$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*" /> </Target> <Target Name="_PrintNames" Condition="$(DryRun)"> - <Exec Command="echo "$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo) -echo "$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo) -echo "$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/* -echo. -echo." /> - </Target> - - <Target Name="_Purge" Condition="$(Purge) and !$(DryRun)"> - <Error Condition="!Exists('$(PythonExe)')" Text="No Python executable available at $(PythonExe)" /> - <Exec Command=""$(PythonExe)" purge.py $(PythonVersion)" /> + <Exec Command="echo "$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo)" /> + <Exec Command="echo "$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo)" /> + <Exec Command="echo "$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*" /> </Target> <Target Name="_TestLayout"> @@ -82,6 +74,7 @@ echo." /> <LogFile>$(OutputPath)\%(Filename)_layoutlog\%(Filename).log</LogFile> </WebInstaller> </ItemGroup> + <Error Text="Could not find installer" Condition="@(WebInstaller) == ''" /> <RemoveDir Directories="%(WebInstaller.SourceDir)" Condition="Exists('%(WebInstaller.SourceDir)')" /> <RemoveDir Directories="%(WebInstaller.LayoutDir)" Condition="Exists('%(WebInstaller.LayoutDir)')" /> <RemoveDir Directories="%(WebInstaller.LogDir)" Condition="Exists('%(WebInstaller.LogDir)')" /> @@ -95,14 +88,22 @@ echo." /> <Message Text="Successfully downloaded %(WebInstaller.Filename)%(WebInstaller.Extension) layout" Importance="high" /> </Target> - <Target Name="Upload" DependsOnTargets="_ValidateProperties;_RunGpg;_PrintNames;_Upload;_Purge" /> + <Target Name="Upload" DependsOnTargets="_ValidateProperties;_RunGpg;_PrintNames;_Upload" /> <Target Name="Test" DependsOnTargets="_TestLayout" /> + <Target Name="Purge"> + <Error Condition="!Exists('$(PythonExe)')" Text="No Python executable available at $(PythonExe)" /> + <Exec Command="echo "$(PythonExe)" purge.py $(PythonVersion)" Condition="$(DryRun)" /> + <Exec Command=""$(PythonExe)" purge.py $(PythonVersion)" Condition="!$(DryRun)" /> + <Message Text="Purged uploaded files" Importance="high" /> + </Target> + <Target Name="ShowHashes"> <ItemGroup> <UserFiles Include="@(File)" Condition="'%(File.CopyTo)' == '$(EXETarget)'" /> </ItemGroup> + <Error Text="No files generated" Condition="@(UserFiles) == ''" /> <Exec Command=""$(PythonExe)" generate_md5.py @(UserFiles->'"%(FullPath)"',' ')" /> </Target>

1 0

bpo-31523: Reliability improvements to the Windows build files (#3900)
by Steve Dower Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/2084b30e540d88b9fc752c5bdcc2f24334… commit: 2084b30e540d88b9fc752c5bdcc2f24334af4f2b branch: master author: Steve Dower <steve.dower(a)microsoft.com> committer: GitHub <noreply(a)github.com> date: 2017-10-05T13:35:36-07:00 summary: bpo-31523: Reliability improvements to the Windows build files (#3900) files: M PCbuild/pyproject.props M Tools/msi/msi.props M Tools/msi/purge.py M Tools/msi/uploadrelease.bat M Tools/msi/uploadrelease.proj diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index aed8f34583b..42ead00d5a1 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -6,7 +6,7 @@ <OutDir>$(BuildPath)</OutDir> <OutDir Condition="!HasTrailingSlash($(OutDir))">$(OutDir)\</OutDir> <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir> - <IntDir>$(Py_IntDir)\$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir> + <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir> <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName> <TargetName>$(TargetName)$(PyDebugExt)</TargetName> <GenerateManifest>false</GenerateManifest> diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index 24a90bd8cb1..0318f7aa8eb 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -48,7 +48,7 @@ </ItemGroup> <PropertyGroup> - <IntermediateOutputPath>$(Py_IntDir)\$(Configuration)_$(Platform)_Setup\$(OutputName)</IntermediateOutputPath> + <IntermediateOutputPath>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\msi_$(OutputName)</IntermediateOutputPath> <IntermediateOutputPath Condition="'$(OutputSuffix)' != ''">$(IntermediateOutputPath)_$(OutputSuffix)</IntermediateOutputPath> <OutputPath Condition="'$(OutputPath)' == ''">$(BuildPath)</OutputPath> <OutputPath Condition="!HasTrailingSlash($(OutputPath))">$(OutputPath)\</OutputPath> diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py index 76dbc49ae82..24fdae24388 100644 --- a/Tools/msi/purge.py +++ b/Tools/msi/purge.py @@ -25,7 +25,7 @@ sys.exit(1) URL = "https://www.python.org/ftp/python/{}/".format(m.group(1)) - +REL = m.group(2) or '' FILES = [ "core.msi", @@ -64,7 +64,7 @@ "python-{}-webinstall.exe".format(m.group(0)), "python-{}-amd64.exe".format(m.group(0)), "python-{}-amd64-webinstall.exe".format(m.group(0)), -] + ["win32{}/{}".format(m.group(2), f) for f in FILES] + ["amd64{}/{}".format(m.group(2), f) for f in FILES] +] + ["win32{}/{}".format(REL, f) for f in FILES] + ["amd64{}/{}".format(REL, f) for f in FILES] print('Purged:') for n in PATHS: diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 4902b6cf3f7..3ae5dbc61bf 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -8,8 +8,9 @@ set HOST= set USER= set TARGET= set DRYRUN=false +set NOUPLOAD= set NOGPG= -set PURGE_OPTION=/p:Purge=true +set NOPURGE= set NOTEST= :CheckOpts @@ -21,8 +22,9 @@ if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts +if "%1" EQU "--skip-upload" (set NOUPLOAD=true) && shift && goto CheckOpts if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts -if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && goto CheckOpts +if "%1" EQU "--skip-purge" (set NOPURGE=true) && shift && goto CheckOpts if "%1" EQU "--skip-test" (set NOTEST=true) && shift && goto CheckOpts if "%1" EQU "-T" (set NOTEST=true) && shift && goto CheckOpts if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 @@ -52,17 +54,43 @@ if defined NOGPG ( call "%PCBUILD%find_msbuild.bat" %MSBUILD% if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) pushd "%D%" -%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% -%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +if not defined NOUPLOAD ( + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 + if errorlevel 1 goto :failed + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false + if errorlevel 1 goto :failed +) +if not defined NOPURGE ( + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge +) if not defined NOTEST ( - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 + call :test x86 + if errorlevel 1 goto :failed + call :test x64 + if errorlevel 1 goto :failed ) %MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 +if errorlevel 1 goto :failed %MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false +if errorlevel 1 goto :failed + popd exit /B 0 +:test +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1 +if errorlevel 1 ( + echo Test failed - purging and retrying + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge + if errorlevel 1 exit /B + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1 +) +exit /B + +:failed +popd +exit /B + :Help echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] echo. diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 75840f2f851..881b825e107 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -10,7 +10,6 @@ <IncludeDoc Condition="'$(IncludeDoc)' == ''">true</IncludeDoc> <BuildForRelease Condition="'$(BuildForRelease)' == ''">true</BuildForRelease> <DryRun Condition="'$(DryRun)' == ''">false</DryRun> - <Purge Condition="'$(Purge)' == ''">false</Purge> </PropertyGroup> <Import Project="msi.props" /> @@ -43,7 +42,8 @@ <Target Name="_RunGpg" Condition="'$(GPG)' != ''" Inputs="@(File)" Outputs="$(IntermediateOutputPath)\gpg\%(FileName)%(Extension).asc"> <MakeDir Directories="$(IntermediateOutputPath)gpg" /> <Delete Files="$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc" Condition="Exists('$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc')" /> - <Exec Command=""$(GPG)" -ba -o "$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc" "%(File.FullPath)"" /> + <Exec Command=""$(GPG)" -ba -o "$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc" "%(File.FullPath)"" + IgnoreExitCode="false" /> <ItemGroup> <File Include="$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc"> <CopyTo>%(File.CopyTo)</CopyTo> @@ -52,23 +52,15 @@ </Target> <Target Name="_Upload" Condition="!$(DryRun)"> - <Exec Command=""$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo) -"$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo) -"$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/* -" /> + <Exec Command=""$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo)" /> + <Exec Command=""$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo)" /> + <Exec Command=""$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*" /> </Target> <Target Name="_PrintNames" Condition="$(DryRun)"> - <Exec Command="echo "$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo) -echo "$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo) -echo "$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/* -echo. -echo." /> - </Target> - - <Target Name="_Purge" Condition="$(Purge) and !$(DryRun)"> - <Error Condition="!Exists('$(PythonExe)')" Text="No Python executable available at $(PythonExe)" /> - <Exec Command=""$(PythonExe)" purge.py $(PythonVersion)" /> + <Exec Command="echo "$(PLINK)" $(User)@$(Host) mkdir %(File.CopyTo) ^&^& chgrp downloads %(File.CopyTo) ^&^& chmod g-w,o+rx %(File.CopyTo)" /> + <Exec Command="echo "$(PSCP)" @(File,' ') $(User)@$(Host):%(File.CopyTo)" /> + <Exec Command="echo "$(PLINK)" $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*" /> </Target> <Target Name="_TestLayout"> @@ -82,6 +74,7 @@ echo." /> <LogFile>$(OutputPath)\%(Filename)_layoutlog\%(Filename).log</LogFile> </WebInstaller> </ItemGroup> + <Error Text="Could not find installer" Condition="@(WebInstaller) == ''" /> <RemoveDir Directories="%(WebInstaller.SourceDir)" Condition="Exists('%(WebInstaller.SourceDir)')" /> <RemoveDir Directories="%(WebInstaller.LayoutDir)" Condition="Exists('%(WebInstaller.LayoutDir)')" /> <RemoveDir Directories="%(WebInstaller.LogDir)" Condition="Exists('%(WebInstaller.LogDir)')" /> @@ -95,14 +88,22 @@ echo." /> <Message Text="Successfully downloaded %(WebInstaller.Filename)%(WebInstaller.Extension) layout" Importance="high" /> </Target> - <Target Name="Upload" DependsOnTargets="_ValidateProperties;_RunGpg;_PrintNames;_Upload;_Purge" /> + <Target Name="Upload" DependsOnTargets="_ValidateProperties;_RunGpg;_PrintNames;_Upload" /> <Target Name="Test" DependsOnTargets="_TestLayout" /> + <Target Name="Purge"> + <Error Condition="!Exists('$(PythonExe)')" Text="No Python executable available at $(PythonExe)" /> + <Exec Command="echo "$(PythonExe)" purge.py $(PythonVersion)" Condition="$(DryRun)" /> + <Exec Command=""$(PythonExe)" purge.py $(PythonVersion)" Condition="!$(DryRun)" /> + <Message Text="Purged uploaded files" Importance="high" /> + </Target> + <Target Name="ShowHashes"> <ItemGroup> <UserFiles Include="@(File)" Condition="'%(File.CopyTo)' == '$(EXETarget)'" /> </ItemGroup> + <Error Text="No files generated" Condition="@(UserFiles) == ''" /> <Exec Command=""$(PythonExe)" generate_md5.py @(UserFiles->'"%(FullPath)"',' ')" /> </Target>

1 0

[2 down, 1 up, 62 flat] Results for Python (master branch) 2017-10-05
by lp_benchmark_robot＠intel.com Oct. 5, 2017

Oct. 5, 2017

Results for project python/master, build date: 2017-10-05 03:02:49-07:00. - commit: c1c47c1 - previous commit: bf477a9 - revision date: 2017-10-05 17:19:26+09:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 1.059% | +0.059% | +4.150% | +7.666% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method| 0.843% | -0.029% | +16.505% | +17.788% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 0.662% | +0.178% | +18.187% | +16.423% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_unknown| 0.509% | +0.202% | +16.871% | +16.435% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 2.056% | +0.077% | +3.723% | +13.504% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 1.550% | -1.119% | +10.401% | +9.115% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 0.656% | -1.182% | +7.004% | +11.569% | +-----+------------------------+--------+------------+------------+------------+ | :-( | crypto_pyaes| 0.827% | -2.850% | +0.753% | +9.657% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 4.500% | +1.024% | +7.992% | +17.862% | +-----+------------------------+--------+------------+------------+------------+ | :-| | django_template| 6.396% | -2.792% | +8.144% | +12.233% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.159% | +0.855% | +4.773% | +7.187% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 0.438% | -1.078% | +5.855% | +5.575% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.676% | -1.160% | +2.616% | +6.692% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 1.364% | +0.310% | +8.641% | +12.215% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 1.507% | +1.515% | +8.316% | +8.480% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 0.682% | -1.097% | +6.052% | +11.420% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 0.484% | -0.800% | +9.104% | +11.479% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 2.849% | -0.466% | +9.210% | +8.276% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 1.951% | +0.363% | +4.741% | +8.284% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 5.404% | -5.844% | -3.315% | +15.960% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_format| 1.154% | -0.153% | +8.628% | +9.084% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 2.631% | +0.632% | +46.735% | +12.183% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.701% | -0.812% | +8.669% | +10.963% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.458% | +0.493% | +17.417% | +12.727% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 1.370% | -0.233% | +7.846% | +9.847% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 2.540% | +0.103% | +4.129% | +4.506% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 0.489% | -0.011% | -0.734% | +0.205% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 0.822% | +0.175% | +2.295% | +8.363% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.446% | +1.371% | +7.285% | +8.673% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 1.125% | -1.642% | -0.624% | +21.698% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_dict| 0.253% | -1.058% | +2.289% | +17.511% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_list| 0.615% | -1.564% | +5.508% | +14.067% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 4.116% | -0.517% | +11.693% | +9.451% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.264% | -0.050% | +0.375% | +9.569% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup| 0.125% | -0.196% | +10.905% | +5.176% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup_no_site| 0.085% | -0.244% | +2.064% | +5.115% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.192% | -1.486% | +9.017% | +13.862% | +-----+------------------------+--------+------------+------------+------------+ | :-) | regex_compile| 5.166% | +11.822% | +3.798% | +9.242% | +-----+------------------------+--------+------------+------------+------------+ | :-( | regex_dna| 0.376% | -3.764% | -1.688% | +11.899% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 3.067% | -0.450% | -2.314% | +4.980% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 2.880% | -2.068% | +9.791% | +4.104% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 1.074% | +0.520% | +7.238% | +14.766% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 0.684% | -2.079% | +0.124% | +4.170% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 1.770% | +0.014% | +26.896% | +9.520% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 3.494% | -2.002% | +4.422% | +5.734% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 1.562% | -1.628% | +14.479% | +8.267% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sparse_mat_mult| 1.681% | -1.787% | +2.190% | -3.294% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 0.364% | -2.443% | +3.619% | +5.395% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 0.948% | +0.320% | +6.488% | +7.182% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 2.992% | -0.252% | +5.942% | +4.860% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlite_synth| 3.899% | +2.201% | +20.133% | +7.719% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.538% | +0.598% | +11.207% | +8.841% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 2.158% | +0.191% | +9.662% | +6.794% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 4.202% | -0.426% | +10.643% | +9.446% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 5.346% | +1.663% | +11.754% | +9.011% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 3.382% | -2.848% | +22.265% | +11.754% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 1.341% | -0.054% | +6.318% | +6.038% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 0.732% | +3.446% | +2.116% | +3.125% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle| 6.644% | -1.453% | +8.726% | +19.567% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 1.478% | -3.087% | -2.840% | +19.805% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 1.685% | -0.233% | +7.855% | +7.023% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 1.088% | -0.260% | +6.369% | +7.435% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 2.732% | +2.048% | +1.908% | +6.764% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 3.402% | +3.408% | -6.535% | +9.436% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.416% | -0.028% | +7.065% | +7.773% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/2-down-1-up-62-flat-results-for-pytho… Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

PEP 553 built-in breakpoint() function (bpo-31353) (#3355)
by Barry Warsaw Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/36c1d1f1e52ba54007cbecb42c5599e5ff… commit: 36c1d1f1e52ba54007cbecb42c5599e5ff62aa52 branch: master author: Barry Warsaw <barry(a)python.org> committer: GitHub <noreply(a)github.com> date: 2017-10-05T12:11:18-04:00 summary: PEP 553 built-in breakpoint() function (bpo-31353) (#3355) Implement PEP 553, built-in breakpoint() with support from sys.breakpointhook(), along with documentation and tests. Closes bpo-31353 files: A Misc/NEWS.d/next/Core and Builtins/2017-09-05-14-19-02.bpo-31353.oGZUeJ.rst M Doc/library/functions.rst M Doc/library/sys.rst M Doc/using/cmdline.rst M Doc/whatsnew/3.7.rst M Lib/test/test_builtin.py M Lib/test/test_inspect.py M Python/bltinmodule.c M Python/sysmodule.c diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 015231d9cff..08093e61fe5 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -7,24 +7,24 @@ Built-in Functions The Python interpreter has a number of functions and types built into it that are always available. They are listed here in alphabetical order. -=================== ================= ================== ================ ==================== -.. .. Built-in Functions .. .. -=================== ================= ================== ================ ==================== -:func:`abs` |func-dict|_ :func:`help` :func:`min` :func:`setattr` -:func:`all` :func:`dir` :func:`hex` :func:`next` :func:`slice` -:func:`any` :func:`divmod` :func:`id` :func:`object` :func:`sorted` -:func:`ascii` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` -:func:`bin` :func:`eval` :func:`int` :func:`open` |func-str|_ -:func:`bool` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` -|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ -:func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` -:func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` -:func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` -:func:`compile` :func:`globals` :func:`map` :func:`reversed` :func:`__import__` +=================== ================= ================== ================== ==================== +.. .. Built-in Functions .. .. +=================== ================= ================== ================== ==================== +:func:`abs` :func:`delattr` :func:`hash` |func-memoryview|_ |func-set|_ +:func:`all` |func-dict|_ :func:`help` :func:`min` :func:`setattr` +:func:`any` :func:`dir` :func:`hex` :func:`next` :func:`slice` +:func:`ascii` :func:`divmod` :func:`id` :func:`object` :func:`sorted` +:func:`bin` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` +:func:`bool` :func:`eval` :func:`int` :func:`open` |func-str|_ +:func:`breakpoint` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` +|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` +|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ +:func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` +:func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` +:func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` +:func:`compile` :func:`globals` :func:`map` :func:`reversed` :func:`__import__` :func:`complex` :func:`hasattr` :func:`max` :func:`round` -:func:`delattr` :func:`hash` |func-memoryview|_ |func-set|_ -=================== ================= ================== ================ ==================== +=================== ================= ================== ================== ==================== .. using :func:`dict` would create a link to another page, so local targets are used, with replacement texts to make the output in the table consistent @@ -113,6 +113,20 @@ are always available. They are listed here in alphabetical order. .. index:: pair: Boolean; type +.. function:: breakpoint(*args, **kws) + + This function drops you into the debugger at the call site. Specifically, + it calls :func:`sys.breakpointhook`, passing ``args`` and ``kws`` straight + through. By default, ``sys.breakpointhook()`` calls + :func:`pdb.set_trace()` expecting no arguments. In this case, it is + purely a convenience function so you don't have to explicitly import + :mod:`pdb` or type as much code to enter the debugger. However, + :func:`sys.breakpointhook` can be set to some other function and + :func:`breakpoint` will automatically call that, allowing you to drop into + the debugger of choice. + + .. versionadded:: 3.7 + .. _func-bytearray: .. class:: bytearray([source[, encoding[, errors]]]) :noindex: diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 85f31368c30..aa7bd477b05 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -109,6 +109,40 @@ always available. This function should be used for internal and specialized purposes only. +.. function:: breakpointhook() + + This hook function is called by built-in :func:`breakpoint`. By default, + it drops you into the :mod:`pdb` debugger, but it can be set to any other + function so that you can choose which debugger gets used. + + The signature of this function is dependent on what it calls. For example, + the default binding (e.g. ``pdb.set_trace()``) expects no arguments, but + you might bind it to a function that expects additional arguments + (positional and/or keyword). The built-in ``breakpoint()`` function passes + its ``*args`` and ``**kws`` straight through. Whatever + ``breakpointhooks()`` returns is returned from ``breakpoint()``. + + The default implementation first consults the environment variable + :envvar:`PYTHONBREAKPOINT`. If that is set to ``"0"`` then this function + returns immediately; i.e. it is a no-op. If the environment variable is + not set, or is set to the empty string, ``pdb.set_trace()`` is called. + Otherwise this variable should name a function to run, using Python's + dotted-import nomenclature, e.g. ``package.subpackage.module.function``. + In this case, ``package.subpackage.module`` would be imported and the + resulting module must have a callable named ``function()``. This is run, + passing in ``*args`` and ``**kws``, and whatever ``function()`` returns, + ``sys.breakpointhook()`` returns to the built-in :func:`breakpoint` + function. + + Note that if anything goes wrong while importing the callable named by + :envvar:`PYTHONBREAKPOINT`, a :exc:`RuntimeWarning` is reported and the + breakpoint is ignored. + + Also note that if ``sys.breakpointhook()`` is overridden programmatically, + :envvar:`PYTHONBREAKPOINT` is *not* consulted. + + .. versionadded:: 3.7 + .. function:: _debugmallocstats() Print low-level information to stderr about the state of CPython's memory @@ -187,14 +221,19 @@ always available. customized by assigning another three-argument function to ``sys.excepthook``. -.. data:: __displayhook__ +.. data:: __breakpointhook__ + __displayhook__ __excepthook__ - These objects contain the original values of ``displayhook`` and ``excepthook`` - at the start of the program. They are saved so that ``displayhook`` and - ``excepthook`` can be restored in case they happen to get replaced with broken + These objects contain the original values of ``breakpointhook``, + ``displayhook``, and ``excepthook`` at the start of the program. They are + saved so that ``breakpointhook``, ``displayhook`` and ``excepthook`` can be + restored in case they happen to get replaced with broken or alternative objects. + .. versionadded:: 3.7 + __breakpointhook__ + .. function:: exc_info() diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 7b60a9b71fe..a8cdff641e4 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -502,6 +502,18 @@ conflict. :option:`-O` multiple times. +.. envvar:: PYTHONBREAKPOINT + + If this is set, it names a callable using dotted-path notation. The module + containing the callable will be imported and then the callable will be run + by the default implementation of :func:`sys.breakpointhook` which itself is + called by built-in :func:`breakpoint`. If not set, or set to the empty + string, it is equivalent to the value "pdb.set_trace". Setting this to the + string "0" causes the default implementation of :func:`sys.breakpointhook` + to do nothing but return immediately. + + .. versionadded:: 3.7 + .. envvar:: PYTHONDEBUG If this is set to a non-empty string it is equivalent to specifying the diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 19b766fd2ff..3e8617ef2a1 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -107,6 +107,25 @@ locale remains active when the core interpreter is initialized. :pep:`538` -- Coercing the legacy C locale to a UTF-8 based locale PEP written and implemented by Nick Coghlan. +.. _whatsnew37-pep553: + +PEP 553: Built-in breakpoint() +------------------------------ + +:pep:`553` describes a new built-in called ``breakpoint()`` which makes it +easy and consistent to enter the Python debugger. Built-in ``breakpoint()`` +calls ``sys.breakpointhook()``. By default, this latter imports ``pdb`` and +then calls ``pdb.set_trace()``, but by binding ``sys.breakpointhook()`` to the +function of your choosing, ``breakpoint()`` can enter any debugger. Or, the +environment variable :envvar:`PYTHONBREAKPOINT` can be set to the callable of +your debugger of choice. Set ``PYTHONBREAKPOINT=0`` to completely disable +built-in ``breakpoint()``. + +.. seealso:: + + :pep:`553` -- Built-in breakpoint() + PEP written and implemented by Barry Warsaw + Other Language Changes ====================== diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 87dcda7b434..0a61c054444 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -17,9 +17,12 @@ import types import unittest import warnings +from contextlib import ExitStack from operator import neg -from test.support import TESTFN, unlink, check_warnings +from test.support import ( + EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink) from test.support.script_helper import assert_python_ok +from unittest.mock import MagicMock, patch try: import pty, signal except ImportError: @@ -1514,6 +1517,111 @@ def test_construct_singletons(self): self.assertRaises(TypeError, tp, 1, 2) self.assertRaises(TypeError, tp, a=1, b=2) + +class TestBreakpoint(unittest.TestCase): + def setUp(self): + # These tests require a clean slate environment. For example, if the + # test suite is run with $PYTHONBREAKPOINT set to something else, it + # will mess up these tests. Similarly for sys.breakpointhook. + # Cleaning the slate here means you can't use breakpoint() to debug + # these tests, but I think that's okay. Just use pdb.set_trace() if + # you must. + self.resources = ExitStack() + self.addCleanup(self.resources.close) + self.env = self.resources.enter_context(EnvironmentVarGuard()) + del self.env['PYTHONBREAKPOINT'] + self.resources.enter_context( + swap_attr(sys, 'breakpointhook', sys.__breakpointhook__)) + + def test_breakpoint(self): + with patch('pdb.set_trace') as mock: + breakpoint() + mock.assert_called_once() + + def test_breakpoint_with_breakpointhook_set(self): + my_breakpointhook = MagicMock() + sys.breakpointhook = my_breakpointhook + breakpoint() + my_breakpointhook.assert_called_once_with() + + def test_breakpoint_with_breakpointhook_reset(self): + my_breakpointhook = MagicMock() + sys.breakpointhook = my_breakpointhook + breakpoint() + my_breakpointhook.assert_called_once_with() + # Reset the hook and it will not be called again. + sys.breakpointhook = sys.__breakpointhook__ + with patch('pdb.set_trace') as mock: + breakpoint() + mock.assert_called_once_with() + my_breakpointhook.assert_called_once_with() + + def test_breakpoint_with_args_and_keywords(self): + my_breakpointhook = MagicMock() + sys.breakpointhook = my_breakpointhook + breakpoint(1, 2, 3, four=4, five=5) + my_breakpointhook.assert_called_once_with(1, 2, 3, four=4, five=5) + + def test_breakpoint_with_passthru_error(self): + def my_breakpointhook(): + pass + sys.breakpointhook = my_breakpointhook + self.assertRaises(TypeError, breakpoint, 1, 2, 3, four=4, five=5) + + @unittest.skipIf(sys.flags.ignore_environment, '-E was given') + def test_envar_good_path_builtin(self): + self.env['PYTHONBREAKPOINT'] = 'int' + with patch('builtins.int') as mock: + breakpoint('7') + mock.assert_called_once_with('7') + + @unittest.skipIf(sys.flags.ignore_environment, '-E was given') + def test_envar_good_path_other(self): + self.env['PYTHONBREAKPOINT'] = 'sys.exit' + with patch('sys.exit') as mock: + breakpoint() + mock.assert_called_once_with() + + @unittest.skipIf(sys.flags.ignore_environment, '-E was given') + def test_envar_good_path_noop_0(self): + self.env['PYTHONBREAKPOINT'] = '0' + with patch('pdb.set_trace') as mock: + breakpoint() + mock.assert_not_called() + + def test_envar_good_path_empty_string(self): + # PYTHONBREAKPOINT='' is the same as it not being set. + self.env['PYTHONBREAKPOINT'] = '' + with patch('pdb.set_trace') as mock: + breakpoint() + mock.assert_called_once_with() + + @unittest.skipIf(sys.flags.ignore_environment, '-E was given') + def test_envar_unimportable(self): + for envar in ( + '.', '..', '.foo', 'foo.', '.int', 'int.' + 'nosuchbuiltin', + 'nosuchmodule.nosuchcallable', + ): + with self.subTest(envar=envar): + self.env['PYTHONBREAKPOINT'] = envar + mock = self.resources.enter_context(patch('pdb.set_trace')) + w = self.resources.enter_context(check_warnings(quiet=True)) + breakpoint() + self.assertEqual( + str(w.message), + f'Ignoring unimportable $PYTHONBREAKPOINT: "{envar}"') + self.assertEqual(w.category, RuntimeWarning) + mock.assert_not_called() + + def test_envar_ignored_when_hook_is_set(self): + self.env['PYTHONBREAKPOINT'] = 'sys.exit' + with patch('sys.exit') as mock: + sys.breakpointhook = int + breakpoint() + mock.assert_not_called() + + @unittest.skipUnless(pty, "the pty and signal modules must be available") class PtyTests(unittest.TestCase): """Tests that use a pseudo terminal to guarantee stdin and stdout are diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 7cc1e78f787..819fcc58537 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3523,7 +3523,8 @@ def test_builtins_have_signatures(self): needs_semantic_update = {"round"} no_signature |= needs_semantic_update # These need *args support in Argument Clinic - needs_varargs = {"min", "max", "print", "__build_class__"} + needs_varargs = {"breakpoint", "min", "max", "print", + "__build_class__"} no_signature |= needs_varargs # These simply weren't covered in the initial AC conversion # for builtin callables diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-05-14-19-02.bpo-31353.oGZUeJ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-05-14-19-02.bpo-31353.oGZUeJ.rst new file mode 100644 index 00000000000..55f81cde8a5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-05-14-19-02.bpo-31353.oGZUeJ.rst @@ -0,0 +1,5 @@ +:pep:`553` - Add a new built-in called ``breakpoint()`` which calls +``sys.breakpointhook()``. By default this imports ``pdb`` and calls +``pdb.set_trace()``, but users may override ``sys.breakpointhook()`` to call +whatever debugger they want. The original value of the hook is saved in +``sys.__breakpointhook__``. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2269fe21657..6215a638c94 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -422,6 +422,28 @@ builtin_callable(PyObject *module, PyObject *obj) return PyBool_FromLong((long)PyCallable_Check(obj)); } +static PyObject * +builtin_breakpoint(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *keywords) +{ + PyObject *hook = PySys_GetObject("breakpointhook"); + + if (hook == NULL) { + PyErr_SetString(PyExc_RuntimeError, "lost sys.breakpointhook"); + return NULL; + } + Py_INCREF(hook); + PyObject *retval = _PyObject_FastCallKeywords(hook, args, nargs, keywords); + Py_DECREF(hook); + return retval; +} + +PyDoc_STRVAR(breakpoint_doc, +"breakpoint(*args, **kws)\n\ +\n\ +Call sys.breakpointhook(*args, **kws). sys.breakpointhook() must accept\n\ +whatever arguments are passed.\n\ +\n\ +By default, this drops you into the pdb debugger."); typedef struct { PyObject_HEAD @@ -2627,6 +2649,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ANY_METHODDEF BUILTIN_ASCII_METHODDEF BUILTIN_BIN_METHODDEF + {"breakpoint", (PyCFunction)builtin_breakpoint, METH_FASTCALL | METH_KEYWORDS, breakpoint_doc}, BUILTIN_CALLABLE_METHODDEF BUILTIN_CHR_METHODDEF BUILTIN_COMPILE_METHODDEF diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6d2cc96b5eb..e38a200c005 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -96,6 +96,81 @@ PySys_SetObject(const char *name, PyObject *v) return PyDict_SetItemString(sd, name, v); } +static PyObject * +sys_breakpointhook(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *keywords) +{ + assert(!PyErr_Occurred()); + char *envar = Py_GETENV("PYTHONBREAKPOINT"); + + if (envar == NULL || strlen(envar) == 0) { + envar = "pdb.set_trace"; + } + else if (!strcmp(envar, "0")) { + /* The breakpoint is explicitly no-op'd. */ + Py_RETURN_NONE; + } + char *last_dot = strrchr(envar, '.'); + char *attrname = NULL; + PyObject *modulepath = NULL; + + if (last_dot == NULL) { + /* The breakpoint is a built-in, e.g. PYTHONBREAKPOINT=int */ + modulepath = PyUnicode_FromString("builtins"); + attrname = envar; + } + else { + /* Split on the last dot; */ + modulepath = PyUnicode_FromStringAndSize(envar, last_dot - envar); + attrname = last_dot + 1; + } + if (modulepath == NULL) { + return NULL; + } + + PyObject *fromlist = Py_BuildValue("(s)", attrname); + if (fromlist == NULL) { + Py_DECREF(modulepath); + return NULL; + } + PyObject *module = PyImport_ImportModuleLevelObject( + modulepath, NULL, NULL, fromlist, 0); + Py_DECREF(modulepath); + Py_DECREF(fromlist); + + if (module == NULL) { + goto error; + } + + PyObject *hook = PyObject_GetAttrString(module, attrname); + Py_DECREF(module); + + if (hook == NULL) { + goto error; + } + PyObject *retval = _PyObject_FastCallKeywords(hook, args, nargs, keywords); + Py_DECREF(hook); + return retval; + + error: + /* If any of the imports went wrong, then warn and ignore. */ + PyErr_Clear(); + int status = PyErr_WarnFormat( + PyExc_RuntimeWarning, 0, + "Ignoring unimportable $PYTHONBREAKPOINT: \"%s\"", envar); + if (status < 0) { + /* Printing the warning raised an exception. */ + return NULL; + } + /* The warning was (probably) issued. */ + Py_RETURN_NONE; +} + +PyDoc_STRVAR(breakpointhook_doc, +"breakpointhook(*args, **kws)\n" +"\n" +"This hook function is called by built-in breakpoint().\n" +); + /* Write repr(o) to sys.stdout using sys.stdout.encoding and 'backslashreplace' error handler. If sys.stdout has a buffer attribute, use sys.stdout.buffer.write(encoded), otherwise redecode the string and use @@ -1365,6 +1440,8 @@ sys_getandroidapilevel(PyObject *self) static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ + {"breakpointhook", (PyCFunction)sys_breakpointhook, + METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, {"callstats", (PyCFunction)sys_callstats, METH_NOARGS, callstats_doc}, {"_clear_type_cache", sys_clear_type_cache, METH_NOARGS, @@ -1977,6 +2054,9 @@ _PySys_BeginInit(void) PyDict_GetItemString(sysdict, "displayhook")); SET_SYS_FROM_STRING_BORROW("__excepthook__", PyDict_GetItemString(sysdict, "excepthook")); + SET_SYS_FROM_STRING_BORROW( + "__breakpointhook__", + PyDict_GetItemString(sysdict, "breakpointhook")); SET_SYS_FROM_STRING("version", PyUnicode_FromString(Py_GetVersion())); SET_SYS_FROM_STRING("hexversion",

1 0

bpo-31556: asyncio.wait_for can cancel futures faster with timeout <= 0 (#3703)
by Yury Selivanov Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/4d071897880b7e84e1a217ebe19971c118… commit: 4d071897880b7e84e1a217ebe19971c118316970 branch: master author: Victor K <hellysmile(a)gmail.com> committer: Yury Selivanov <yury(a)magic.io> date: 2017-10-05T12:04:39-04:00 summary: bpo-31556: asyncio.wait_for can cancel futures faster with timeout <= 0 (#3703) files: A Misc/NEWS.d/next/Library/2017-09-22-23-48-49.bpo-31556.9J0u5H.rst M Lib/asyncio/tasks.py M Lib/test/test_asyncio/test_tasks.py diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 575d205404a..52fef181cec 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -334,6 +334,15 @@ def wait_for(fut, timeout, *, loop=None): if timeout is None: return (yield from fut) + if timeout <= 0: + fut = ensure_future(fut, loop=loop) + + if fut.done(): + return fut.result() + + fut.cancel() + raise futures.TimeoutError() + waiter = loop.create_future() timeout_handle = loop.call_later(timeout, _release_waiter, waiter) cb = functools.partial(_release_waiter, waiter) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 36082ec7c6a..7ff56b560b6 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -661,6 +661,76 @@ def task(): t.cancel() self.assertRaises(asyncio.CancelledError, loop.run_until_complete, t) + def test_wait_for_timeout_less_then_0_or_0_future_done(self): + def gen(): + when = yield + self.assertAlmostEqual(0, when) + + loop = self.new_test_loop(gen) + + fut = self.new_future(loop) + fut.set_result('done') + + ret = loop.run_until_complete(asyncio.wait_for(fut, 0, loop=loop)) + + self.assertEqual(ret, 'done') + self.assertTrue(fut.done()) + self.assertAlmostEqual(0, loop.time()) + + def test_wait_for_timeout_less_then_0_or_0_coroutine_do_not_started(self): + def gen(): + when = yield + self.assertAlmostEqual(0, when) + + loop = self.new_test_loop(gen) + + foo_started = False + + @asyncio.coroutine + def foo(): + nonlocal foo_started + foo_started = True + + with self.assertRaises(asyncio.TimeoutError): + loop.run_until_complete(asyncio.wait_for(foo(), 0, loop=loop)) + + self.assertAlmostEqual(0, loop.time()) + self.assertEqual(foo_started, False) + + def test_wait_for_timeout_less_then_0_or_0(self): + def gen(): + when = yield + self.assertAlmostEqual(0.2, when) + when = yield 0 + self.assertAlmostEqual(0, when) + + for timeout in [0, -1]: + with self.subTest(timeout=timeout): + loop = self.new_test_loop(gen) + + foo_running = None + + @asyncio.coroutine + def foo(): + nonlocal foo_running + foo_running = True + try: + yield from asyncio.sleep(0.2, loop=loop) + finally: + foo_running = False + return 'done' + + fut = self.new_task(loop, foo()) + + with self.assertRaises(asyncio.TimeoutError): + loop.run_until_complete(asyncio.wait_for( + fut, timeout, loop=loop)) + self.assertTrue(fut.done()) + # it should have been cancelled due to the timeout + self.assertTrue(fut.cancelled()) + self.assertAlmostEqual(0, loop.time()) + self.assertEqual(foo_running, False) + def test_wait_for(self): def gen(): diff --git a/Misc/NEWS.d/next/Library/2017-09-22-23-48-49.bpo-31556.9J0u5H.rst b/Misc/NEWS.d/next/Library/2017-09-22-23-48-49.bpo-31556.9J0u5H.rst new file mode 100644 index 00000000000..2e6b0284696 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-22-23-48-49.bpo-31556.9J0u5H.rst @@ -0,0 +1 @@ +Cancel asyncio.wait_for future faster if timeout <= 0

1 0

[2.7] remove tabs from getcompiler.c (GH-3892) (#3894)
by Benjamin Peterson Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/2fb8e9385944276884c2a2dc5d6275a04e… commit: 2fb8e9385944276884c2a2dc5d6275a04e04c879 branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Benjamin Peterson <benjamin(a)python.org> date: 2017-10-05T08:41:31-07:00 summary: [2.7] remove tabs from getcompiler.c (GH-3892) (#3894) (cherry picked from commit a8ed11742b4c2115597977ce04fa8e043d9e0792) files: M Python/getcompiler.c diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 0f441deb84d..9d9c33ac2de 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -24,5 +24,5 @@ const char * Py_GetCompiler(void) { - return COMPILER; + return COMPILER; }

1 0

[3.6] remove tabs from getcompiler.c (GH-3892) (#3893)
by Benjamin Peterson Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/8aafd5fd7961df1068325d41cc0acd72e7… commit: 8aafd5fd7961df1068325d41cc0acd72e7009a0f branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Benjamin Peterson <benjamin(a)python.org> date: 2017-10-05T08:41:06-07:00 summary: [3.6] remove tabs from getcompiler.c (GH-3892) (#3893) (cherry picked from commit a8ed11742b4c2115597977ce04fa8e043d9e0792) files: M Python/getcompiler.c diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 0f441deb84d..9d9c33ac2de 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -24,5 +24,5 @@ const char * Py_GetCompiler(void) { - return COMPILER; + return COMPILER; }

1 0

[3.6] bpo-31178: Mock os.waitpid() in test_subprocess (GH-3896) (#3897)
by Victor Stinner Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/fae0512e58619231a566bf77aa21148440… commit: fae0512e58619231a566bf77aa21148440b0ec8d branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Victor Stinner <victor.stinner(a)gmail.com> date: 2017-10-05T07:10:59-07:00 summary: [3.6] bpo-31178: Mock os.waitpid() in test_subprocess (GH-3896) (#3897) Fix test_exception_errpipe_bad_data() and test_exception_errpipe_normal() of test_subprocess: mock os.waitpid() to avoid calling the real os.waitpid(0, 0) which is an unexpected side effect of the test. (cherry picked from commit 11045c9d8a21dd9bd182a3939189db02815f9783) files: M Lib/test/test_subprocess.py diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 391d08cd9c3..5016513decd 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1568,8 +1568,10 @@ def proper_error(*args): fork_exec.side_effect = proper_error - with self.assertRaises(IsADirectoryError): - self.PopenNoDestructor(["non_existent_command"]) + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(IsADirectoryError): + self.PopenNoDestructor(["non_existent_command"]) @mock.patch("subprocess._posixsubprocess.fork_exec") def test_exception_errpipe_bad_data(self, fork_exec): @@ -1586,8 +1588,10 @@ def bad_error(*args): fork_exec.side_effect = bad_error - with self.assertRaises(subprocess.SubprocessError) as e: - self.PopenNoDestructor(["non_existent_command"]) + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(subprocess.SubprocessError) as e: + self.PopenNoDestructor(["non_existent_command"]) self.assertIn(repr(error_data), str(e.exception))

1 0

bpo-31178: Mock os.waitpid() in test_subprocess (#3896)
by Victor Stinner Oct. 5, 2017

Oct. 5, 2017

https://github.com/python/cpython/commit/11045c9d8a21dd9bd182a3939189db0281… commit: 11045c9d8a21dd9bd182a3939189db02815f9783 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-10-05T06:32:53-07:00 summary: bpo-31178: Mock os.waitpid() in test_subprocess (#3896) Fix test_exception_errpipe_bad_data() and test_exception_errpipe_normal() of test_subprocess: mock os.waitpid() to avoid calling the real os.waitpid(0, 0) which is an unexpected side effect of the test. files: M Lib/test/test_subprocess.py diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 00dc37bc2c7..3ba5c028517 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1559,8 +1559,10 @@ def proper_error(*args): fork_exec.side_effect = proper_error - with self.assertRaises(IsADirectoryError): - self.PopenNoDestructor(["non_existent_command"]) + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(IsADirectoryError): + self.PopenNoDestructor(["non_existent_command"]) @mock.patch("subprocess._posixsubprocess.fork_exec") def test_exception_errpipe_bad_data(self, fork_exec): @@ -1577,8 +1579,10 @@ def bad_error(*args): fork_exec.side_effect = bad_error - with self.assertRaises(subprocess.SubprocessError) as e: - self.PopenNoDestructor(["non_existent_command"]) + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(subprocess.SubprocessError) as e: + self.PopenNoDestructor(["non_existent_command"]) self.assertIn(repr(error_data), str(e.exception))

1 0