Mailman 3 September 2017 - Python-checkins

bpo-31460: Simplify the API of IDLE's Module Browser. (#3842)
by Terry Jan Reedy 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/d6bb65f378e34fe0c11fdb39588357ecf2… commit: d6bb65f378e34fe0c11fdb39588357ecf22964eb branch: master author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-09-30T19:54:28-04:00 summary: bpo-31460: Simplify the API of IDLE's Module Browser. (#3842) Passing a widget instead of an flist with a root widget opens the option of creating a browser frame that is only part of a window. Passing a full file name instead of pieces assumed to come from a .py file opens the possibility of browsing python files that do not end in .py. files: A Misc/NEWS.d/next/IDLE/2017-09-30-19-03-26.bpo-31460.HpveI6.rst M Lib/idlelib/browser.py M Lib/idlelib/editor.py M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 9b323155913..a185ed584ab 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -5,9 +5,8 @@ - reparse when source changed (maybe just a button would be OK?) (or recheck on window popup) - add popup menu with more options (e.g. doc strings, base classes, imports) -- show function argument list? (have to do pattern matching on source) -- should the classes and methods lists also be in the module's menu bar? - add base classes to class browser tree +- finish removing limitation to x.py files (ModuleBrowserTreeItem) """ import os @@ -58,19 +57,18 @@ def transform_children(child_dict, modname=None): class ModuleBrowser: """Browse module classes and functions in IDLE. """ - # This class is the base class for pathbrowser.PathBrowser. + # This class is also the base class for pathbrowser.PathBrowser. # Init and close are inherited, other methods are overriden. + # PathBrowser.__init__ does not call __init__ below. - def __init__(self, flist, name, path, *, _htest=False, _utest=False): - # XXX This API should change, if the file doesn't end in ".py" - # XXX the code here is bogus! + def __init__(self, master, path, *, _htest=False, _utest=False): """Create a window for browsing a module's structure. Args: - flist: filelist.FileList instance used as the root for the window. - name: Python module to parse. - path: Module search path. - _htest - bool, change box when location running htest. + master: parent for widgets. + path: full path of file to browse. + _htest - bool; change box location when running htest. + -utest - bool; suppress contents when running unittest. Global variables: file_open: Function used for opening a file. @@ -84,35 +82,36 @@ def __init__(self, flist, name, path, *, _htest=False, _utest=False): global file_open if not (_htest or _utest): file_open = pyshell.flist.open - self.name = name - self.file = os.path.join(path[0], self.name + ".py") + self.master = master + self.path = path self._htest = _htest self._utest = _utest - self.init(flist) + self.init() def close(self, event=None): "Dismiss the window and the tree nodes." self.top.destroy() self.node.destroy() - def init(self, flist): + def init(self): "Create browser tkinter widgets, including the tree." - self.flist = flist + root = self.master # reset pyclbr pyclbr._modules.clear() # create top - self.top = top = ListedToplevel(flist.root) + self.top = top = ListedToplevel(root) top.protocol("WM_DELETE_WINDOW", self.close) top.bind("<Escape>", self.close) if self._htest: # place dialog below parent if running htest top.geometry("+%d+%d" % - (flist.root.winfo_rootx(), flist.root.winfo_rooty() + 200)) + (root.winfo_rootx(), root.winfo_rooty() + 200)) self.settitle() top.focus_set() # create scrolled canvas theme = idleConf.CurrentTheme() background = idleConf.GetHighlight(theme, 'normal')['background'] - sc = ScrolledCanvas(top, bg=background, highlightthickness=0, takefocus=1) + sc = ScrolledCanvas(top, bg=background, highlightthickness=0, + takefocus=1) sc.frame.pack(expand=1, fill="both") item = self.rootnode() self.node = node = TreeNode(sc.canvas, None, item) @@ -122,18 +121,19 @@ def init(self, flist): def settitle(self): "Set the window title." - self.top.wm_title("Module Browser - " + self.name) + self.top.wm_title("Module Browser - " + os.path.basename(self.path)) self.top.wm_iconname("Module Browser") def rootnode(self): "Return a ModuleBrowserTreeItem as the root of the tree." - return ModuleBrowserTreeItem(self.file) + return ModuleBrowserTreeItem(self.path) class ModuleBrowserTreeItem(TreeItem): """Browser tree for Python module. Uses TreeItem as the basis for the structure of the tree. + Used by both browsers. """ def __init__(self, file): @@ -170,8 +170,8 @@ def IsExpandable(self): def listchildren(self): "Return sequenced classes and functions in the module." - dir, file = os.path.split(self.file) - name, ext = os.path.splitext(file) + dir, base = os.path.split(self.file) + name, ext = os.path.splitext(base) if os.path.normcase(ext) != ".py": return [] try: @@ -227,25 +227,22 @@ def OnDoubleClick(self): def _module_browser(parent): # htest # - try: - file = sys.argv[1] # If pass file on command line - # If this succeeds, unittest will fail. - except IndexError: + if len(sys.argv) > 1: # If pass file on command line. + file = sys.argv[1] + else: file = __file__ - # Add objects for htest + # Add nested objects for htest. class Nested_in_func(TreeNode): def nested_in_class(): pass def closure(): class Nested_in_closure: pass - dir, file = os.path.split(file) - name = os.path.splitext(file)[0] - flist = pyshell.PyShellFileList(parent) global file_open - file_open = flist.open - ModuleBrowser(flist, name, [dir], _htest=True) + file_open = pyshell.PyShellFileList(parent).open + ModuleBrowser(parent, file, _htest=True) if __name__ == "__main__": - from unittest import main - main('idlelib.idle_test.test_browser', verbosity=2, exit=False) + if len(sys.argv) == 1: # If pass file on command line, unittest fails. + from unittest import main + main('idlelib.idle_test.test_browser', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_module_browser) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 5b16ccee8a6..87d1eef01ec 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -664,10 +664,8 @@ def open_module_browser(self, event=None): filename = self.open_module() if filename is None: return "break" - head, tail = os.path.split(filename) - base, ext = os.path.splitext(tail) from idlelib import browser - browser.ModuleBrowser(self.flist, base, [head]) + browser.ModuleBrowser(self.root, filename) return "break" def open_path_browser(self, event=None): diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index a4add89b93b..59e03c5aab3 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -24,30 +24,24 @@ def setUpClass(cls): requires('gui') cls.root = Tk() cls.root.withdraw() - cls.flist = filelist.FileList(cls.root) - cls.file = __file__ - cls.path = os.path.dirname(cls.file) - cls.module = os.path.basename(cls.file).rstrip('.py') - cls.mb = browser.ModuleBrowser(cls.flist, cls.module, [cls.path], _utest=True) + cls.mb = browser.ModuleBrowser(cls.root, __file__, _utest=True) @classmethod def tearDownClass(cls): cls.mb.close() cls.root.destroy() - del cls.root, cls.flist, cls.mb + del cls.root, cls.mb def test_init(self): mb = self.mb eq = self.assertEqual - eq(mb.name, self.module) - eq(mb.file, self.file) - eq(mb.flist, self.flist) + eq(mb.path, __file__) eq(pyclbr._modules, {}) self.assertIsInstance(mb.node, TreeNode) def test_settitle(self): mb = self.mb - self.assertIn(self.module, mb.top.title()) + self.assertIn(os.path.basename(__file__), mb.top.title()) self.assertEqual(mb.top.iconname(), 'Module Browser') def test_rootnode(self): diff --git a/Misc/NEWS.d/next/IDLE/2017-09-30-19-03-26.bpo-31460.HpveI6.rst b/Misc/NEWS.d/next/IDLE/2017-09-30-19-03-26.bpo-31460.HpveI6.rst new file mode 100644 index 00000000000..6492115260a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-30-19-03-26.bpo-31460.HpveI6.rst @@ -0,0 +1,6 @@ +Simplify the API of IDLE's Module Browser. + +Passing a widget instead of an flist with a root widget opens the option of +creating a browser frame that is only part of a window. Passing a full file +name instead of pieces assumed to come from a .py file opens the possibility +of browsing python files that do not end in .py.

1 0

[3.6] bpo-31649: Make IDLE's _htest, _utest parameters keyword-only. (GH-3839) (#3841)
by Terry Jan Reedy 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/40c54d5e1aaab91cb7df71f735112d20b5… commit: 40c54d5e1aaab91cb7df71f735112d20b5e5b755 branch: 3.6 author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-09-30T18:45:53-04:00 summary: [3.6] bpo-31649: Make IDLE's _htest, _utest parameters keyword-only. (GH-3839) (#3841) (cherry picked from commit bfebfd8) files: A Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst M Lib/idlelib/browser.py M Lib/idlelib/config_key.py M Lib/idlelib/configdialog.py M Lib/idlelib/help_about.py M Lib/idlelib/pathbrowser.py M Lib/idlelib/textview.py diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 145ecd1b525..010d8c34982 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -61,7 +61,7 @@ class ModuleBrowser: # This class is the base class for pathbrowser.PathBrowser. # Init and close are inherited, other methods are overriden. - def __init__(self, flist, name, path, _htest=False, _utest=False): + def __init__(self, flist, name, path, *, _htest=False, _utest=False): # XXX This API should change, if the file doesn't end in ".py" # XXX the code here is bogus! """Create a window for browsing a module's structure. diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 5556b767964..3a865f869a0 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -14,7 +14,7 @@ class GetKeysDialog(Toplevel): keyerror_title = 'Key Sequence Error' def __init__(self, parent, title, action, currentKeySequences, - _htest=False, _utest=False): + *, _htest=False, _utest=False): """ action - string, the name of the virtual event these keys will be mapped to diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index a05f3b957af..d27b0727d3d 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -41,7 +41,7 @@ class ConfigDialog(Toplevel): """Config dialog for IDLE. """ - def __init__(self, parent, title='', _htest=False, _utest=False): + def __init__(self, parent, title='', *, _htest=False, _utest=False): """Show the tabbed dialog for user configuration. Args: diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index 679ac78d32d..77b4b189620 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -23,7 +23,7 @@ class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title=None, _htest=False, _utest=False): + def __init__(self, parent, title=None, *, _htest=False, _utest=False): """Create popup, do not return until tk widget destroyed. parent - parent of this dialog diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index b0c8c6d3055..c0aa2a15909 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -9,7 +9,7 @@ class PathBrowser(ModuleBrowser): - def __init__(self, flist, _htest=False, _utest=False): + def __init__(self, flist, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index de4b190ff2e..e3b55065c6d 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -57,7 +57,7 @@ class ViewWindow(Toplevel): "A simple text viewer dialog for IDLE." def __init__(self, parent, title, text, modal=True, - _htest=False, _utest=False): + *, _htest=False, _utest=False): """Show the given text in a scrollable window with a 'close' button. If modal is left True, users cannot interact with other windows diff --git a/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst b/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst new file mode 100644 index 00000000000..cc995862372 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst @@ -0,0 +1 @@ +IDLE - Make _htest, _utest parameters keyword only.

1 0

bpo-31649: Make IDLE's _htest, _utest parameters keyword-only. (#3839)
by Terry Jan Reedy 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/bfebfd81de21b7906df386fce845f2b1f5… commit: bfebfd81de21b7906df386fce845f2b1f5ffd212 branch: master author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-09-30T17:37:53-04:00 summary: bpo-31649: Make IDLE's _htest, _utest parameters keyword-only. (#3839) files: A Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst M Lib/idlelib/browser.py M Lib/idlelib/config_key.py M Lib/idlelib/configdialog.py M Lib/idlelib/help_about.py M Lib/idlelib/pathbrowser.py M Lib/idlelib/textview.py diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 603d299518d..9b323155913 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -61,7 +61,7 @@ class ModuleBrowser: # This class is the base class for pathbrowser.PathBrowser. # Init and close are inherited, other methods are overriden. - def __init__(self, flist, name, path, _htest=False, _utest=False): + def __init__(self, flist, name, path, *, _htest=False, _utest=False): # XXX This API should change, if the file doesn't end in ".py" # XXX the code here is bogus! """Create a window for browsing a module's structure. diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 5556b767964..3a865f869a0 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -14,7 +14,7 @@ class GetKeysDialog(Toplevel): keyerror_title = 'Key Sequence Error' def __init__(self, parent, title, action, currentKeySequences, - _htest=False, _utest=False): + *, _htest=False, _utest=False): """ action - string, the name of the virtual event these keys will be mapped to diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index a05f3b957af..d27b0727d3d 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -41,7 +41,7 @@ class ConfigDialog(Toplevel): """Config dialog for IDLE. """ - def __init__(self, parent, title='', _htest=False, _utest=False): + def __init__(self, parent, title='', *, _htest=False, _utest=False): """Show the tabbed dialog for user configuration. Args: diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index 679ac78d32d..77b4b189620 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -23,7 +23,7 @@ class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title=None, _htest=False, _utest=False): + def __init__(self, parent, title=None, *, _htest=False, _utest=False): """Create popup, do not return until tk widget destroyed. parent - parent of this dialog diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index b0c8c6d3055..c0aa2a15909 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -9,7 +9,7 @@ class PathBrowser(ModuleBrowser): - def __init__(self, flist, _htest=False, _utest=False): + def __init__(self, flist, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index de4b190ff2e..e3b55065c6d 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -57,7 +57,7 @@ class ViewWindow(Toplevel): "A simple text viewer dialog for IDLE." def __init__(self, parent, title, text, modal=True, - _htest=False, _utest=False): + *, _htest=False, _utest=False): """Show the given text in a scrollable window with a 'close' button. If modal is left True, users cannot interact with other windows diff --git a/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst b/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst new file mode 100644 index 00000000000..cc995862372 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst @@ -0,0 +1 @@ +IDLE - Make _htest, _utest parameters keyword only.

1 0

[2.7] bpo-31627: Make test_mailbox be lenient to empty hostname. (GH-3821) (#3838)
by Serhiy Storchaka 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/1163fb9be08c6072455b59d6f954e60662… commit: 1163fb9be08c6072455b59d6f954e60662d18e75 branch: 2.7 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-09-30T22:52:25+03:00 summary: [2.7] bpo-31627: Make test_mailbox be lenient to empty hostname. (GH-3821) (#3838) (cherry picked from commit f4ea642cb60556231e714089a79d3c59c202661e) files: M Lib/test/test_mailbox.py diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 2261bb854cb..f8db5d5723f 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -654,7 +654,7 @@ def test_create_tmp(self, repetitions=10): hostname = hostname.replace(':', r'\072') pid = os.getpid() pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)" - r"Q(?P<Q>\d+)\.(?P<host>[^:/]+)") + r"Q(?P<Q>\d+)\.(?P<host>[^:/]*)") previous_groups = None for x in xrange(repetitions): tmp_file = self._box._create_tmp()

1 0

[3.6] bpo-31627: Make test_mailbox be lenient to empty hostname. (GH-3821) (#3837)
by Serhiy Storchaka 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/d9c21a45fdc83904753d91ad45e3eafa19… commit: d9c21a45fdc83904753d91ad45e3eafa195f4713 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-09-30T22:52:00+03:00 summary: [3.6] bpo-31627: Make test_mailbox be lenient to empty hostname. (GH-3821) (#3837) (cherry picked from commit f4ea642cb60556231e714089a79d3c59c202661e) files: M Lib/test/test_mailbox.py diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 2ba944335ae..3807b95b158 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -746,7 +746,7 @@ def test_create_tmp(self, repetitions=10): hostname = hostname.replace(':', r'\072') pid = os.getpid() pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)" - r"Q(?P<Q>\d+)\.(?P<host>[^:/]+)") + r"Q(?P<Q>\d+)\.(?P<host>[^:/]*)") previous_groups = None for x in range(repetitions): tmp_file = self._box._create_tmp()

1 0

[3.6] bpo-31592: Fix an assertion failure in Python parser in case of a bad unicodedata.normalize(). (GH-3767) (#3836)
by Serhiy Storchaka 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/a4dfe1c9eaca6faf206f63a178233ffea2… commit: a4dfe1c9eaca6faf206f63a178233ffea208c906 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-09-30T22:51:37+03:00 summary: [3.6] bpo-31592: Fix an assertion failure in Python parser in case of a bad unicodedata.normalize(). (GH-3767) (#3836) (cherry picked from commit 7dc46d8cf5854d9f4ce3271b29c21aea4872e8ad) files: A Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst M Lib/test/test_ast.py M Python/ast.c diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8c62408bd8a..e68d0de2f8b 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -421,6 +421,16 @@ def test_empty_yield_from(self): compile(empty_yield_from, "<test>", "exec") self.assertIn("field value is required", str(cm.exception)) + @support.cpython_only + def test_issue31592(self): + # There shouldn't be an assertion failure in case of a bad + # unicodedata.normalize(). + import unicodedata + def bad_normalize(*args): + return None + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') + class ASTHelpers_Test(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst new file mode 100644 index 00000000000..29f3461921e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst @@ -0,0 +1,2 @@ +Fixed an assertion failure in Python parser in case of a bad `unicodedata.normalize()`. +Patch by Oren Milman. diff --git a/Python/ast.c b/Python/ast.c index aa4acc9b261..2973b9fd866 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -589,7 +589,6 @@ struct compiling { PyArena *c_arena; /* Arena for allocating memory. */ PyObject *c_filename; /* filename */ PyObject *c_normalize; /* Normalization function from unicodedata. */ - PyObject *c_normalize_args; /* Normalization argument tuple. */ }; static asdl_seq *seq_for_testlist(struct compiling *, const node *); @@ -624,12 +623,6 @@ init_normalization(struct compiling *c) Py_DECREF(m); if (!c->c_normalize) return 0; - c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None); - if (!c->c_normalize_args) { - Py_CLEAR(c->c_normalize); - return 0; - } - PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL); return 1; } @@ -645,15 +638,29 @@ new_identifier(const char *n, struct compiling *c) identifier; if so, normalize to NFKC. */ if (!PyUnicode_IS_ASCII(id)) { PyObject *id2; + _Py_IDENTIFIER(NFKC); if (!c->c_normalize && !init_normalization(c)) { Py_DECREF(id); return NULL; } - PyTuple_SET_ITEM(c->c_normalize_args, 1, id); - id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL); + PyObject *form = _PyUnicode_FromId(&PyId_NFKC); + if (form == NULL) { + Py_DECREF(id); + return NULL; + } + PyObject *args[2] = {form, id}; + id2 = _PyObject_FastCall(c->c_normalize, args, 2); Py_DECREF(id); if (!id2) return NULL; + if (!PyUnicode_Check(id2)) { + PyErr_Format(PyExc_TypeError, + "unicodedata.normalize() must return a string, not " + "%.200s", + Py_TYPE(id2)->tp_name); + Py_DECREF(id2); + return NULL; + } id = id2; } PyUnicode_InternInPlace(&id); @@ -773,7 +780,6 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, /* borrowed reference */ c.c_filename = filename; c.c_normalize = NULL; - c.c_normalize_args = NULL; if (TYPE(n) == encoding_decl) n = CHILD(n, 0); @@ -866,8 +872,6 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, out: if (c.c_normalize) { Py_DECREF(c.c_normalize); - PyTuple_SET_ITEM(c.c_normalize_args, 1, NULL); - Py_DECREF(c.c_normalize_args); } return res; }

1 0

bpo-31592: Fix an assertion failure in Python parser in case of a bad unicodedata.normalize(). (#3767)
by Serhiy Storchaka 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/7dc46d8cf5854d9f4ce3271b29c21aea48… commit: 7dc46d8cf5854d9f4ce3271b29c21aea4872e8ad branch: master author: Oren Milman <orenmn(a)gmail.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-09-30T20:16:24+03:00 summary: bpo-31592: Fix an assertion failure in Python parser in case of a bad unicodedata.normalize(). (#3767) files: A Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst M Lib/test/test_ast.py M Python/ast.c diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 2f59527cb8a..aa53503e3b5 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -431,6 +431,16 @@ def test_empty_yield_from(self): compile(empty_yield_from, "<test>", "exec") self.assertIn("field value is required", str(cm.exception)) + @support.cpython_only + def test_issue31592(self): + # There shouldn't be an assertion failure in case of a bad + # unicodedata.normalize(). + import unicodedata + def bad_normalize(*args): + return None + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') + class ASTHelpers_Test(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst new file mode 100644 index 00000000000..29f3461921e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst @@ -0,0 +1,2 @@ +Fixed an assertion failure in Python parser in case of a bad `unicodedata.normalize()`. +Patch by Oren Milman. diff --git a/Python/ast.c b/Python/ast.c index fd42c0073de..33356da0756 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -594,7 +594,6 @@ struct compiling { PyArena *c_arena; /* Arena for allocating memory. */ PyObject *c_filename; /* filename */ PyObject *c_normalize; /* Normalization function from unicodedata. */ - PyObject *c_normalize_args; /* Normalization argument tuple. */ }; static asdl_seq *seq_for_testlist(struct compiling *, const node *); @@ -631,12 +630,6 @@ init_normalization(struct compiling *c) Py_DECREF(m); if (!c->c_normalize) return 0; - c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None); - if (!c->c_normalize_args) { - Py_CLEAR(c->c_normalize); - return 0; - } - PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL); return 1; } @@ -652,15 +645,29 @@ new_identifier(const char *n, struct compiling *c) identifier; if so, normalize to NFKC. */ if (!PyUnicode_IS_ASCII(id)) { PyObject *id2; + _Py_IDENTIFIER(NFKC); if (!c->c_normalize && !init_normalization(c)) { Py_DECREF(id); return NULL; } - PyTuple_SET_ITEM(c->c_normalize_args, 1, id); - id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL); + PyObject *form = _PyUnicode_FromId(&PyId_NFKC); + if (form == NULL) { + Py_DECREF(id); + return NULL; + } + PyObject *args[2] = {form, id}; + id2 = _PyObject_FastCall(c->c_normalize, args, 2); Py_DECREF(id); if (!id2) return NULL; + if (!PyUnicode_Check(id2)) { + PyErr_Format(PyExc_TypeError, + "unicodedata.normalize() must return a string, not " + "%.200s", + Py_TYPE(id2)->tp_name); + Py_DECREF(id2); + return NULL; + } id = id2; } PyUnicode_InternInPlace(&id); @@ -779,7 +786,6 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, /* borrowed reference */ c.c_filename = filename; c.c_normalize = NULL; - c.c_normalize_args = NULL; if (TYPE(n) == encoding_decl) n = CHILD(n, 0); @@ -872,8 +878,6 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, out: if (c.c_normalize) { Py_DECREF(c.c_normalize); - PyTuple_SET_ITEM(c.c_normalize_args, 1, NULL); - Py_DECREF(c.c_normalize_args); } return res; }

1 0

bpo-31627: Make test_mailbox be lenient to empty hostname. (#3821)
by Serhiy Storchaka 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/f4ea642cb60556231e714089a79d3c59c2… commit: f4ea642cb60556231e714089a79d3c59c202661e branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-09-30T17:10:57+03:00 summary: bpo-31627: Make test_mailbox be lenient to empty hostname. (#3821) files: M Lib/test/test_mailbox.py diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 2ba944335ae..3807b95b158 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -746,7 +746,7 @@ def test_create_tmp(self, repetitions=10): hostname = hostname.replace(':', r'\072') pid = os.getpid() pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)" - r"Q(?P<Q>\d+)\.(?P<host>[^:/]+)") + r"Q(?P<Q>\d+)\.(?P<host>[^:/]*)") previous_groups = None for x in range(repetitions): tmp_file = self._box._create_tmp()

1 0

[2.7] bpo-31285: Don't raise a SystemError in warnings.warn_explicit() in case __loader__.get_source() has a bad splitlines() method. (GH-3219) (#3823)
by Serhiy Storchaka 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/40d736bcf40c10aa5fc7d6cc305a6641af… commit: 40d736bcf40c10aa5fc7d6cc305a6641afd3cd0e branch: 2.7 author: Oren Milman <orenmn(a)gmail.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-09-30T17:06:55+03:00 summary: [2.7] bpo-31285: Don't raise a SystemError in warnings.warn_explicit() in case __loader__.get_source() has a bad splitlines() method. (GH-3219) (#3823) (cherry picked from commit 91fb0af) files: M Lib/test/test_warnings.py M Python/_warnings.c diff --git a/Lib/test/test_warnings.py b/Lib/test/test_warnings.py index 0023363832c..ae081eeee3f 100644 --- a/Lib/test/test_warnings.py +++ b/Lib/test/test_warnings.py @@ -584,6 +584,27 @@ def test_stderr_none(self): self.assertNotIn(b'Warning!', stderr) self.assertNotIn(b'Error', stderr) + def test_issue31285(self): + # warn_explicit() shouldn't raise a SystemError in case the return + # value of get_source() has a bad splitlines() method. + class BadLoader: + def get_source(self, fullname): + class BadSource(str): + def splitlines(self): + return 42 + return BadSource('spam') + + wmod = self.module + with original_warnings.catch_warnings(module=wmod): + wmod.filterwarnings('default', category=UserWarning) + + with test_support.captured_stderr() as stderr: + wmod.warn_explicit( + 'foo', UserWarning, 'bar', 1, + module_globals={'__loader__': BadLoader(), + '__name__': 'foobar'}) + self.assertIn('UserWarning: foo', stderr.getvalue()) + @test_support.cpython_only def test_issue31411(self): # warn_explicit() shouldn't raise a SystemError in case diff --git a/Python/_warnings.c b/Python/_warnings.c index dd168f92593..37875e56199 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -684,8 +684,9 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) } /* Split the source into lines. */ - source_list = PyObject_CallMethodObjArgs(source, splitlines_name, - NULL); + source_list = PyObject_CallMethodObjArgs((PyObject *)&PyString_Type, + splitlines_name, source, + NULL); Py_DECREF(source); if (!source_list) return NULL;

1 0

bpo-31648: Improve ElementPath (#3835)
by Serhiy Storchaka 30 Sep '17

30 Sep '17

https://github.com/python/cpython/commit/101a5e84acbab9d880e150195f23185dfb… commit: 101a5e84acbab9d880e150195f23185dfb5449a9 branch: master author: scoder <stefan_ml(a)behnel.de> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-09-30T16:35:21+03:00 summary: bpo-31648: Improve ElementPath (#3835) * Allow whitespace inside of ElementPath predicates. * Add ElementPath predicate support for text comparison of the current node, like "[.='text']". files: A Misc/NEWS.d/next/Library/2017-09-30-10-45-12.bpo-31648.Cai7ji.rst M Doc/library/xml.etree.elementtree.rst M Doc/whatsnew/3.7.rst M Lib/test/test_xml_etree.py M Lib/xml/etree/ElementPath.py diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 7d814ad406e..61808596a55 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -437,6 +437,11 @@ Supported XPath syntax | ``[tag]`` | Selects all elements that have a child named | | | ``tag``. Only immediate children are supported. | +-----------------------+------------------------------------------------------+ +| ``[.='text']`` | Selects all elements whose complete text content, | +| | including descendants, equals the given ``text``. | +| | | +| | .. versionadded:: 3.7 | ++-----------------------+------------------------------------------------------+ | ``[tag='text']`` | Selects all elements that have a child named | | | ``tag`` whose complete text content, including | | | descendants, equals the given ``text``. | diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a474e767529..845ed643f97 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -281,6 +281,14 @@ Function :func:`~uu.encode` now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) +xml.etree +--------- + +:ref:`ElementPath <elementtree-xpath>` predicates in the :meth:`find` +methods can now compare text of the current node with ``[. = "text"]``, +not only text in children. Predicates also allow adding spaces for +better readability. (Contributed by Stefan Behnel in :issue:`31648`.) + zipapp ------ diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 661ad8b9d4d..02812f32bc9 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2237,6 +2237,39 @@ def test_findall(self): ['tag'] * 2) self.assertEqual(e.findall('section//'), e.findall('section//*')) + self.assertEqual(summarize_list(e.findall(".//section[tag='subtext']")), + ['section']) + self.assertEqual(summarize_list(e.findall(".//section[tag ='subtext']")), + ['section']) + self.assertEqual(summarize_list(e.findall(".//section[tag= 'subtext']")), + ['section']) + self.assertEqual(summarize_list(e.findall(".//section[tag = 'subtext']")), + ['section']) + self.assertEqual(summarize_list(e.findall(".//section[ tag = 'subtext' ]")), + ['section']) + + self.assertEqual(summarize_list(e.findall(".//tag[.='subtext']")), + ['tag']) + self.assertEqual(summarize_list(e.findall(".//tag[. ='subtext']")), + ['tag']) + self.assertEqual(summarize_list(e.findall('.//tag[.= "subtext"]')), + ['tag']) + self.assertEqual(summarize_list(e.findall('.//tag[ . = "subtext" ]')), + ['tag']) + self.assertEqual(summarize_list(e.findall(".//tag[. = 'subtext']")), + ['tag']) + self.assertEqual(summarize_list(e.findall(".//tag[. = 'subtext ']")), + []) + self.assertEqual(summarize_list(e.findall(".//tag[.= ' subtext']")), + []) + + # duplicate section => 2x tag matches + e[1] = e[2] + self.assertEqual(summarize_list(e.findall(".//section[tag = 'subtext']")), + ['section', 'section']) + self.assertEqual(summarize_list(e.findall(".//tag[. = 'subtext']")), + ['tag', 'tag']) + def test_test_find_with_ns(self): e = ET.XML(SAMPLE_XML_NS) self.assertEqual(summarize_list(e.findall('tag')), []) diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index 361f6d54fa5..c9d6ef345b9 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -157,6 +157,9 @@ def prepare_predicate(next, token): return if token[0] == "]": break + if token == ('', ''): + # ignore whitespace + continue if token[0] and token[0][:1] in "'\"": token = "'", token[0][1:-1] signature.append(token[0] or "-") @@ -188,16 +191,22 @@ def select(context, result): if elem.find(tag) is not None: yield elem return select - if signature == "-='" and not re.match(r"\-?\d+$", predicate[0]): - # [tag='value'] + if signature == ".='" or (signature == "-='" and not re.match(r"\-?\d+$", predicate[0])): + # [.='value'] or [tag='value'] tag = predicate[0] value = predicate[-1] - def select(context, result): - for elem in result: - for e in elem.findall(tag): - if "".join(e.itertext()) == value: + if tag: + def select(context, result): + for elem in result: + for e in elem.findall(tag): + if "".join(e.itertext()) == value: + yield elem + break + else: + def select(context, result): + for elem in result: + if "".join(elem.itertext()) == value: yield elem - break return select if signature == "-" or signature == "-()" or signature == "-()-": # [index] or [last()] or [last()-index] diff --git a/Misc/NEWS.d/next/Library/2017-09-30-10-45-12.bpo-31648.Cai7ji.rst b/Misc/NEWS.d/next/Library/2017-09-30-10-45-12.bpo-31648.Cai7ji.rst new file mode 100644 index 00000000000..8b39ce92fee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-30-10-45-12.bpo-31648.Cai7ji.rst @@ -0,0 +1,6 @@ +Improvements to path predicates in ElementTree: + +* Allow whitespace around predicate parts, i.e. "[a = 'text']" instead of requiring the less readable "[a='text']". +* Add support for text comparison of the current node, like "[.='text']". + +Patch by Stefan Behnel.

1 0