Mailman 3 February 2020 - Python-checkins

bpo-13790: Change 'string' to 'specification' in format doc (GH-18690)
by Terry Jan Reedy 28 Feb '20

28 Feb '20

https://github.com/python/cpython/commit/916895f93905f8b8dad677cceff501833f… commit: 916895f93905f8b8dad677cceff501833f5a633a branch: master author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2020-02-28T14:59:16-05:00 summary: bpo-13790: Change 'string' to 'specification' in format doc (GH-18690) files: A Misc/NEWS.d/next/Documentation/2020-02-28-14-39-25.bpo-13790.hvLaRI.rst M Doc/library/string.rst diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 89c169a512b52..fa906f799c108 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -302,9 +302,9 @@ specification is to be interpreted. Most built-in types implement the following options for format specifications, although some of the formatting options are only supported by the numeric types. -A general convention is that an empty format string (``""``) produces +A general convention is that an empty format specification produces the same result as if you had called :func:`str` on the value. A -non-empty format string typically modifies the result. +non-empty format specification typically modifies the result. The general form of a *standard format specifier* is: diff --git a/Misc/NEWS.d/next/Documentation/2020-02-28-14-39-25.bpo-13790.hvLaRI.rst b/Misc/NEWS.d/next/Documentation/2020-02-28-14-39-25.bpo-13790.hvLaRI.rst new file mode 100644 index 0000000000000..77db173168fc5 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2020-02-28-14-39-25.bpo-13790.hvLaRI.rst @@ -0,0 +1 @@ +Change 'string' to 'specification' in format doc.

1 0

bpo-39781: Do not jump when select in IDLE codecontext (GH-18683)
by Miss Islington (bot) 28 Feb '20

28 Feb '20

https://github.com/python/cpython/commit/846ca4961da24669e6e0c901986e66ff48… commit: 846ca4961da24669e6e0c901986e66ff485917b2 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-28T10:43:25-08:00 summary: bpo-39781: Do not jump when select in IDLE codecontext (GH-18683) Previously, the button-up part of selecting with a mouse was treated as a click that meant 'jump' to this line, which modified the context and undid the selection (cherry picked from commit c705fd1e89ccb8f6d414ec817b4616546147d877) Co-authored-by: Terry Jan Reedy <tjreedy(a)udel.edu> files: A Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/codecontext.py M Lib/idlelib/idle_test/test_codecontext.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index fe1706ed5a8be..134f843d85e8a 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,8 @@ Released on 2019-12-16? ====================================== +bpo-39781: Selecting code context lines no longer causes a jump. + bpo-39663: Add tests for pyparse find_good_parse_start(). bpo-39600: Remove duplicate font names from configuration list. diff --git a/Lib/idlelib/codecontext.py b/Lib/idlelib/codecontext.py index 4ce98136fe417..989b30e599465 100644 --- a/Lib/idlelib/codecontext.py +++ b/Lib/idlelib/codecontext.py @@ -7,7 +7,6 @@ enclosing block. The number of hint lines is determined by the maxlines variable in the codecontext section of config-extensions.def. Lines which do not open blocks are not shown in the context hints pane. - """ import re from sys import maxsize as INFINITY @@ -17,8 +16,8 @@ from idlelib.config import idleConf -BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", - "if", "try", "while", "with", "async"} +BLOCKOPENERS = {'class', 'def', 'if', 'elif', 'else', 'while', 'for', + 'try', 'except', 'finally', 'with', 'async'} def get_spaces_firstword(codeline, c=re.compile(r"^(\s*)(\w*)")): @@ -84,7 +83,7 @@ def __del__(self): if self.t1 is not None: try: self.text.after_cancel(self.t1) - except tkinter.TclError: + except tkinter.TclError: # pragma: no cover pass self.t1 = None @@ -112,7 +111,7 @@ def toggle_code_context_event(self, event=None): padx += widget.tk.getint(info['padx']) padx += widget.tk.getint(widget.cget('padx')) border += widget.tk.getint(widget.cget('border')) - self.context = tkinter.Text( + context = self.context = tkinter.Text( self.editwin.text_frame, height=1, width=1, # Don't request more than we get. @@ -120,11 +119,11 @@ def toggle_code_context_event(self, event=None): padx=padx, border=border, relief=SUNKEN, state='disabled') self.update_font() self.update_highlight_colors() - self.context.bind('<ButtonRelease-1>', self.jumptoline) + context.bind('<ButtonRelease-1>', self.jumptoline) # Get the current context and initiate the recurring update event. self.timer_event() # Grid the context widget above the text widget. - self.context.grid(row=0, column=1, sticky=NSEW) + context.grid(row=0, column=1, sticky=NSEW) line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'linenumber') @@ -215,18 +214,25 @@ def update_code_context(self): self.context['state'] = 'disabled' def jumptoline(self, event=None): - "Show clicked context line at top of editor." - lines = len(self.info) - if lines == 1: # No context lines are showing. - newtop = 1 - else: - # Line number clicked. - contextline = int(float(self.context.index('insert'))) - # Lines not displayed due to maxlines. - offset = max(1, lines - self.context_depth) - 1 - newtop = self.info[offset + contextline][0] - self.text.yview(f'{newtop}.0') - self.update_code_context() + """ Show clicked context line at top of editor. + + If a selection was made, don't jump; allow copying. + If no visible context, show the top line of the file. + """ + try: + self.context.index("sel.first") + except tkinter.TclError: + lines = len(self.info) + if lines == 1: # No context lines are showing. + newtop = 1 + else: + # Line number clicked. + contextline = int(float(self.context.index('insert'))) + # Lines not displayed due to maxlines. + offset = max(1, lines - self.context_depth) - 1 + newtop = self.info[offset + contextline][0] + self.text.yview(f'{newtop}.0') + self.update_code_context() def timer_event(self): "Event on editor text widget triggered every UPDATEINTERVAL ms." diff --git a/Lib/idlelib/idle_test/test_codecontext.py b/Lib/idlelib/idle_test/test_codecontext.py index 3ec49e97af6f9..9578cc731a6f9 100644 --- a/Lib/idlelib/idle_test/test_codecontext.py +++ b/Lib/idlelib/idle_test/test_codecontext.py @@ -332,6 +332,14 @@ def test_jumptoline(self): jump() eq(cc.topvisible, 8) + # Context selection stops jump. + cc.text.yview('5.0') + cc.update_code_context() + cc.context.tag_add('sel', '1.0', '2.0') + cc.context.mark_set('insert', '1.0') + jump() # Without selection, to line 2. + eq(cc.topvisible, 5) + @mock.patch.object(codecontext.CodeContext, 'update_code_context') def test_timer_event(self, mock_update): # Ensure code context is not active. diff --git a/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst b/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst new file mode 100644 index 0000000000000..4ae0defc2e217 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst @@ -0,0 +1 @@ +Selecting code context lines no longer causes a jump.

1 0

bpo-39781: Do not jump when select in IDLE codecontext (GH-18683)
by Miss Islington (bot) 28 Feb '20

28 Feb '20

https://github.com/python/cpython/commit/f4198aee4c288ce47c5803e87a461e31f8… commit: f4198aee4c288ce47c5803e87a461e31f81a0138 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-28T10:41:17-08:00 summary: bpo-39781: Do not jump when select in IDLE codecontext (GH-18683) Previously, the button-up part of selecting with a mouse was treated as a click that meant 'jump' to this line, which modified the context and undid the selection (cherry picked from commit c705fd1e89ccb8f6d414ec817b4616546147d877) Co-authored-by: Terry Jan Reedy <tjreedy(a)udel.edu> files: A Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/codecontext.py M Lib/idlelib/idle_test/test_codecontext.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 3c3367244852a..0fa08962b4107 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,8 @@ Released on 2019-12-16? ====================================== +bpo-39781: Selecting code context lines no longer causes a jump. + bpo-39663: Add tests for pyparse find_good_parse_start(). bpo-39600: Remove duplicate font names from configuration list. diff --git a/Lib/idlelib/codecontext.py b/Lib/idlelib/codecontext.py index 4ce98136fe417..989b30e599465 100644 --- a/Lib/idlelib/codecontext.py +++ b/Lib/idlelib/codecontext.py @@ -7,7 +7,6 @@ enclosing block. The number of hint lines is determined by the maxlines variable in the codecontext section of config-extensions.def. Lines which do not open blocks are not shown in the context hints pane. - """ import re from sys import maxsize as INFINITY @@ -17,8 +16,8 @@ from idlelib.config import idleConf -BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", - "if", "try", "while", "with", "async"} +BLOCKOPENERS = {'class', 'def', 'if', 'elif', 'else', 'while', 'for', + 'try', 'except', 'finally', 'with', 'async'} def get_spaces_firstword(codeline, c=re.compile(r"^(\s*)(\w*)")): @@ -84,7 +83,7 @@ def __del__(self): if self.t1 is not None: try: self.text.after_cancel(self.t1) - except tkinter.TclError: + except tkinter.TclError: # pragma: no cover pass self.t1 = None @@ -112,7 +111,7 @@ def toggle_code_context_event(self, event=None): padx += widget.tk.getint(info['padx']) padx += widget.tk.getint(widget.cget('padx')) border += widget.tk.getint(widget.cget('border')) - self.context = tkinter.Text( + context = self.context = tkinter.Text( self.editwin.text_frame, height=1, width=1, # Don't request more than we get. @@ -120,11 +119,11 @@ def toggle_code_context_event(self, event=None): padx=padx, border=border, relief=SUNKEN, state='disabled') self.update_font() self.update_highlight_colors() - self.context.bind('<ButtonRelease-1>', self.jumptoline) + context.bind('<ButtonRelease-1>', self.jumptoline) # Get the current context and initiate the recurring update event. self.timer_event() # Grid the context widget above the text widget. - self.context.grid(row=0, column=1, sticky=NSEW) + context.grid(row=0, column=1, sticky=NSEW) line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'linenumber') @@ -215,18 +214,25 @@ def update_code_context(self): self.context['state'] = 'disabled' def jumptoline(self, event=None): - "Show clicked context line at top of editor." - lines = len(self.info) - if lines == 1: # No context lines are showing. - newtop = 1 - else: - # Line number clicked. - contextline = int(float(self.context.index('insert'))) - # Lines not displayed due to maxlines. - offset = max(1, lines - self.context_depth) - 1 - newtop = self.info[offset + contextline][0] - self.text.yview(f'{newtop}.0') - self.update_code_context() + """ Show clicked context line at top of editor. + + If a selection was made, don't jump; allow copying. + If no visible context, show the top line of the file. + """ + try: + self.context.index("sel.first") + except tkinter.TclError: + lines = len(self.info) + if lines == 1: # No context lines are showing. + newtop = 1 + else: + # Line number clicked. + contextline = int(float(self.context.index('insert'))) + # Lines not displayed due to maxlines. + offset = max(1, lines - self.context_depth) - 1 + newtop = self.info[offset + contextline][0] + self.text.yview(f'{newtop}.0') + self.update_code_context() def timer_event(self): "Event on editor text widget triggered every UPDATEINTERVAL ms." diff --git a/Lib/idlelib/idle_test/test_codecontext.py b/Lib/idlelib/idle_test/test_codecontext.py index 3ec49e97af6f9..9578cc731a6f9 100644 --- a/Lib/idlelib/idle_test/test_codecontext.py +++ b/Lib/idlelib/idle_test/test_codecontext.py @@ -332,6 +332,14 @@ def test_jumptoline(self): jump() eq(cc.topvisible, 8) + # Context selection stops jump. + cc.text.yview('5.0') + cc.update_code_context() + cc.context.tag_add('sel', '1.0', '2.0') + cc.context.mark_set('insert', '1.0') + jump() # Without selection, to line 2. + eq(cc.topvisible, 5) + @mock.patch.object(codecontext.CodeContext, 'update_code_context') def test_timer_event(self, mock_update): # Ensure code context is not active. diff --git a/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst b/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst new file mode 100644 index 0000000000000..4ae0defc2e217 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst @@ -0,0 +1 @@ +Selecting code context lines no longer causes a jump.

1 0

bpo-39781: Do not jump when select in IDLE codecontext (GH-18683)
by Terry Jan Reedy 28 Feb '20

28 Feb '20

https://github.com/python/cpython/commit/c705fd1e89ccb8f6d414ec817b46165461… commit: c705fd1e89ccb8f6d414ec817b4616546147d877 branch: master author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2020-02-28T13:22:55-05:00 summary: bpo-39781: Do not jump when select in IDLE codecontext (GH-18683) Previously, the button-up part of selecting with a mouse was treated as a click that meant 'jump' to this line, which modified the context and undid the selection files: A Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/codecontext.py M Lib/idlelib/idle_test/test_codecontext.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 021e1f7710e0f..0651b3d68dc8b 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,8 @@ Released on 2020-10-05? ====================================== +bpo-39781: Selecting code context lines no longer causes a jump. + bpo-39663: Add tests for pyparse find_good_parse_start(). bpo-39600: Remove duplicate font names from configuration list. diff --git a/Lib/idlelib/codecontext.py b/Lib/idlelib/codecontext.py index 4ce98136fe417..989b30e599465 100644 --- a/Lib/idlelib/codecontext.py +++ b/Lib/idlelib/codecontext.py @@ -7,7 +7,6 @@ enclosing block. The number of hint lines is determined by the maxlines variable in the codecontext section of config-extensions.def. Lines which do not open blocks are not shown in the context hints pane. - """ import re from sys import maxsize as INFINITY @@ -17,8 +16,8 @@ from idlelib.config import idleConf -BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", - "if", "try", "while", "with", "async"} +BLOCKOPENERS = {'class', 'def', 'if', 'elif', 'else', 'while', 'for', + 'try', 'except', 'finally', 'with', 'async'} def get_spaces_firstword(codeline, c=re.compile(r"^(\s*)(\w*)")): @@ -84,7 +83,7 @@ def __del__(self): if self.t1 is not None: try: self.text.after_cancel(self.t1) - except tkinter.TclError: + except tkinter.TclError: # pragma: no cover pass self.t1 = None @@ -112,7 +111,7 @@ def toggle_code_context_event(self, event=None): padx += widget.tk.getint(info['padx']) padx += widget.tk.getint(widget.cget('padx')) border += widget.tk.getint(widget.cget('border')) - self.context = tkinter.Text( + context = self.context = tkinter.Text( self.editwin.text_frame, height=1, width=1, # Don't request more than we get. @@ -120,11 +119,11 @@ def toggle_code_context_event(self, event=None): padx=padx, border=border, relief=SUNKEN, state='disabled') self.update_font() self.update_highlight_colors() - self.context.bind('<ButtonRelease-1>', self.jumptoline) + context.bind('<ButtonRelease-1>', self.jumptoline) # Get the current context and initiate the recurring update event. self.timer_event() # Grid the context widget above the text widget. - self.context.grid(row=0, column=1, sticky=NSEW) + context.grid(row=0, column=1, sticky=NSEW) line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'linenumber') @@ -215,18 +214,25 @@ def update_code_context(self): self.context['state'] = 'disabled' def jumptoline(self, event=None): - "Show clicked context line at top of editor." - lines = len(self.info) - if lines == 1: # No context lines are showing. - newtop = 1 - else: - # Line number clicked. - contextline = int(float(self.context.index('insert'))) - # Lines not displayed due to maxlines. - offset = max(1, lines - self.context_depth) - 1 - newtop = self.info[offset + contextline][0] - self.text.yview(f'{newtop}.0') - self.update_code_context() + """ Show clicked context line at top of editor. + + If a selection was made, don't jump; allow copying. + If no visible context, show the top line of the file. + """ + try: + self.context.index("sel.first") + except tkinter.TclError: + lines = len(self.info) + if lines == 1: # No context lines are showing. + newtop = 1 + else: + # Line number clicked. + contextline = int(float(self.context.index('insert'))) + # Lines not displayed due to maxlines. + offset = max(1, lines - self.context_depth) - 1 + newtop = self.info[offset + contextline][0] + self.text.yview(f'{newtop}.0') + self.update_code_context() def timer_event(self): "Event on editor text widget triggered every UPDATEINTERVAL ms." diff --git a/Lib/idlelib/idle_test/test_codecontext.py b/Lib/idlelib/idle_test/test_codecontext.py index 3ec49e97af6f9..9578cc731a6f9 100644 --- a/Lib/idlelib/idle_test/test_codecontext.py +++ b/Lib/idlelib/idle_test/test_codecontext.py @@ -332,6 +332,14 @@ def test_jumptoline(self): jump() eq(cc.topvisible, 8) + # Context selection stops jump. + cc.text.yview('5.0') + cc.update_code_context() + cc.context.tag_add('sel', '1.0', '2.0') + cc.context.mark_set('insert', '1.0') + jump() # Without selection, to line 2. + eq(cc.topvisible, 5) + @mock.patch.object(codecontext.CodeContext, 'update_code_context') def test_timer_event(self, mock_update): # Ensure code context is not active. diff --git a/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst b/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst new file mode 100644 index 0000000000000..4ae0defc2e217 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2020-02-27-22-17-09.bpo-39781.bbYBeL.rst @@ -0,0 +1 @@ +Selecting code context lines no longer causes a jump.

1 0

bpo-12915: Improve Unicode support for package names and attributes. (GH-18517)
by Vinay Sajip 28 Feb '20

28 Feb '20

https://github.com/python/cpython/commit/4f17c5cd9a1ec50fe8de7ef68c39220a01… commit: 4f17c5cd9a1ec50fe8de7ef68c39220a01a862cb branch: master author: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> committer: GitHub <noreply(a)github.com> date: 2020-02-28T14:26:27Z summary: bpo-12915: Improve Unicode support for package names and attributes. (GH-18517) files: M Lib/pkgutil.py M Lib/test/test_pkgutil.py diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 4bc3083ac197e..4c184678a2912 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -638,8 +638,8 @@ def get_data(package, resource): return loader.get_data(resource_name) -_DOTTED_WORDS = r'[a-z_]\w*(\.[a-z_]\w*)*' -_NAME_PATTERN = re.compile(f'^({_DOTTED_WORDS})(:({_DOTTED_WORDS})?)?$', re.I) +_DOTTED_WORDS = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' +_NAME_PATTERN = re.compile(f'^(?P<pkg>{_DOTTED_WORDS})(?P<cln>:(?P<obj>{_DOTTED_WORDS})?)?$', re.U) del _DOTTED_WORDS def resolve_name(name): @@ -677,11 +677,12 @@ def resolve_name(name): m = _NAME_PATTERN.match(name) if not m: raise ValueError(f'invalid format: {name!r}') - groups = m.groups() - if groups[2]: + gd = m.groupdict() + if gd.get('cln'): # there is a colon - a one-step import is all that's needed - mod = importlib.import_module(groups[0]) - parts = groups[3].split('.') if groups[3] else [] + mod = importlib.import_module(gd['pkg']) + parts = gd.get('obj') + parts = parts.split('.') if parts else [] else: # no colon - have to iterate to find the package boundary parts = name.split('.') diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 906150b10495b..53456c2f7659e 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -229,8 +229,40 @@ def test_name_resolution(self): ('logging.handlers:SysLogHandler.NO_SUCH_VALUE', AttributeError), ('logging.handlers.SysLogHandler.NO_SUCH_VALUE', AttributeError), ('ZeroDivisionError', ImportError), + ('os.path.9abc', ValueError), + ('9abc', ValueError), ) + # add some Unicode package names to the mix. + + unicode_words = ('\u0935\u092e\u0938', + '\xe9', '\xc8', + '\uc548\ub155\ud558\uc138\uc694', + '\u3055\u3088\u306a\u3089', + '\u3042\u308a\u304c\u3068\u3046', + '\u0425\u043e\u0440\u043e\u0448\u043e', + '\u0441\u043f\u0430\u0441\u0438\u0431\u043e', + '\u73b0\u4ee3\u6c49\u8bed\u5e38\u7528\u5b57\u8868') + + for uw in unicode_words: + d = os.path.join(self.dirname, uw) + os.makedirs(d, exist_ok=True) + # make an empty __init__.py file + f = os.path.join(d, '__init__.py') + with open(f, 'w') as f: + f.write('') + f.flush() + # now import the package we just created; clearing the caches is + # needed, otherwise the newly created package isn't found + importlib.invalidate_caches() + mod = importlib.import_module(uw) + success_cases += (uw, mod), + if len(uw) > 1: + failure_cases += (uw[:-1], ImportError), + + # add an example with a Unicode digit at the start + failure_cases += ('\u0966\u0935\u092e\u0938', ValueError), + for s, expected in success_cases: with self.subTest(s=s): o = pkgutil.resolve_name(s)

1 0

Fuzz struct.unpack and catch RecursionError in re.compile (GH-18679)
by Ammar Askar 28 Feb '20

28 Feb '20

https://github.com/python/cpython/commit/e263bb1e97ae8f84fb4f2ab5b0c4f529a2… commit: e263bb1e97ae8f84fb4f2ab5b0c4f529a2e5696d branch: master author: Ammar Askar <ammar(a)ammaraskar.com> committer: GitHub <noreply(a)github.com> date: 2020-02-27T23:05:02-08:00 summary: Fuzz struct.unpack and catch RecursionError in re.compile (GH-18679) files: A Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/hello_string A Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/long_zero A Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/varied_format_string M Modules/_xxtestfuzz/fuzz_tests.txt M Modules/_xxtestfuzz/fuzzer.c diff --git a/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/hello_string b/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/hello_string new file mode 100644 index 0000000000000..92d47cd358eef Binary files /dev/null and b/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/hello_string differ diff --git a/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/long_zero b/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/long_zero new file mode 100644 index 0000000000000..d952225c3a6e0 Binary files /dev/null and b/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/long_zero differ diff --git a/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/varied_format_string b/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/varied_format_string new file mode 100644 index 0000000000000..a150dc087adfe Binary files /dev/null and b/Modules/_xxtestfuzz/fuzz_struct_unpack_corpus/varied_format_string differ diff --git a/Modules/_xxtestfuzz/fuzz_tests.txt b/Modules/_xxtestfuzz/fuzz_tests.txt index 9d330a668ee88..053b77b41b111 100644 --- a/Modules/_xxtestfuzz/fuzz_tests.txt +++ b/Modules/_xxtestfuzz/fuzz_tests.txt @@ -5,3 +5,4 @@ fuzz_json_loads fuzz_sre_compile fuzz_sre_match fuzz_csv_reader +fuzz_struct_unpack diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 74ba819b8b50b..6bd2c3aedccc9 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -79,6 +79,69 @@ static int fuzz_builtin_unicode(const char* data, size_t size) { return 0; } + +PyObject* struct_unpack_method = NULL; +PyObject* struct_error = NULL; +/* Called by LLVMFuzzerTestOneInput for initialization */ +static int init_struct_unpack() { + /* Import struct.unpack */ + PyObject* struct_module = PyImport_ImportModule("struct"); + if (struct_module == NULL) { + return 0; + } + struct_error = PyObject_GetAttrString(struct_module, "error"); + if (struct_error == NULL) { + return 0; + } + struct_unpack_method = PyObject_GetAttrString(struct_module, "unpack"); + return struct_unpack_method != NULL; +} +/* Fuzz struct.unpack(x, y) */ +static int fuzz_struct_unpack(const char* data, size_t size) { + /* Everything up to the first null byte is considered the + format. Everything after is the buffer */ + const char* first_null = memchr(data, '\0', size); + if (first_null == NULL) { + return 0; + } + + size_t format_length = first_null - data; + size_t buffer_length = size - format_length - 1; + + PyObject* pattern = PyBytes_FromStringAndSize(data, format_length); + if (pattern == NULL) { + return 0; + } + PyObject* buffer = PyBytes_FromStringAndSize(first_null + 1, buffer_length); + if (buffer == NULL) { + Py_DECREF(pattern); + return 0; + } + + PyObject* unpacked = PyObject_CallFunctionObjArgs( + struct_unpack_method, pattern, buffer, NULL); + /* Ignore any overflow errors, these are easily triggered accidentally */ + if (unpacked == NULL && PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Clear(); + } + /* The pascal format string will throw a negative size when passing 0 + like: struct.unpack('0p', b'') */ + if (unpacked == NULL && PyErr_ExceptionMatches(PyExc_SystemError)) { + PyErr_Clear(); + } + /* Ignore any struct.error exceptions, these can be caused by invalid + formats or incomplete buffers both of which are common. */ + if (unpacked == NULL && PyErr_ExceptionMatches(struct_error)) { + PyErr_Clear(); + } + + Py_XDECREF(unpacked); + Py_DECREF(pattern); + Py_DECREF(buffer); + return 0; +} + + #define MAX_JSON_TEST_SIZE 0x10000 PyObject* json_loads_method = NULL; @@ -190,9 +253,10 @@ static int fuzz_sre_compile(const char* data, size_t size) { PyErr_Clear(); } /* Ignore some common errors thrown by sre_parse: - Overflow, Assertion and Index */ + Overflow, Assertion, Recursion and Index */ if (compiled == NULL && (PyErr_ExceptionMatches(PyExc_OverflowError) || PyErr_ExceptionMatches(PyExc_AssertionError) || + PyErr_ExceptionMatches(PyExc_RecursionError) || PyErr_ExceptionMatches(PyExc_IndexError)) ) { PyErr_Clear(); @@ -378,6 +442,16 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_builtin_unicode) rv |= _run_fuzz(data, size, fuzz_builtin_unicode); #endif +#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_struct_unpack) + static int STRUCT_UNPACK_INITIALIZED = 0; + if (!STRUCT_UNPACK_INITIALIZED && !init_struct_unpack()) { + PyErr_Print(); + abort(); + } else { + STRUCT_UNPACK_INITIALIZED = 1; + } + rv |= _run_fuzz(data, size, fuzz_struct_unpack); +#endif #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_json_loads) static int JSON_LOADS_INITIALIZED = 0; if (!JSON_LOADS_INITIALIZED && !init_json_loads()) {

1 0

closes bpo-39721: Fix constness of members of tok_state struct. (GH-18600)
by Andy Lester 27 Feb '20

27 Feb '20

https://github.com/python/cpython/commit/384f3c536dd15ba33ea7e8afb4087ae359… commit: 384f3c536dd15ba33ea7e8afb4087ae359d4c12e branch: master author: Andy Lester <andy(a)petdance.com> committer: GitHub <noreply(a)github.com> date: 2020-02-27T18:44:52-08:00 summary: closes bpo-39721: Fix constness of members of tok_state struct. (GH-18600) The function PyTokenizer_FromUTF8 from Parser/tokenizer.c had a comment: /* XXX: constify members. */ This patch addresses that. In the tok_state struct: * end and start were non-const but could be made const * str and input were const but should have been non-const Changes to support this include: * decode_str() now returns a char * since it is allocated. * PyTokenizer_FromString() and PyTokenizer_FromUTF8() each creates a new char * for an allocate string instead of reusing the input const char *. * PyTokenizer_Get() and tok_get() now take const char ** arguments. * Various local vars are const or non-const accordingly. I was able to remove five casts that cast away constness. files: M Parser/parsetok.c M Parser/tokenizer.c M Parser/tokenizer.h diff --git a/Parser/parsetok.c b/Parser/parsetok.c index b0b1bd38a7bba..554455dbc2bad 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -240,7 +240,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, #endif for (;;) { - char *a, *b; + const char *a, *b; int type; size_t len; char *str; @@ -371,7 +371,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, buffer after parsing. Trailing whitespace and comments are OK. */ if (err_ret->error == E_DONE && start == single_input) { - char *cur = tok->cur; + const char *cur = tok->cur; char c = *tok->cur; for (;;) { diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 630b0aaab03f9..f82b102998171 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -59,7 +59,9 @@ tok_new(void) sizeof(struct tok_state)); if (tok == NULL) return NULL; - tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL; + tok->buf = tok->cur = tok->inp = NULL; + tok->start = NULL; + tok->end = NULL; tok->done = E_OK; tok->fp = NULL; tok->input = NULL; @@ -111,7 +113,9 @@ error_ret(struct tok_state *tok) /* XXX */ tok->decoding_erred = 1; if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */ PyMem_FREE(tok->buf); - tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL; + tok->buf = tok->cur = tok->inp = NULL; + tok->start = NULL; + tok->end = NULL; tok->done = E_DECODE; return NULL; /* as if it were EOF */ } @@ -664,11 +668,11 @@ translate_newlines(const char *s, int exec_input, struct tok_state *tok) { Look for encoding declarations inside STR, and record them inside TOK. */ -static const char * +static char * decode_str(const char *input, int single, struct tok_state *tok) { PyObject* utf8 = NULL; - const char *str; + char *str; const char *s; const char *newl[2] = {NULL, NULL}; int lineno = 0; @@ -726,16 +730,18 @@ struct tok_state * PyTokenizer_FromString(const char *str, int exec_input) { struct tok_state *tok = tok_new(); + char *decoded; + if (tok == NULL) return NULL; - str = decode_str(str, exec_input, tok); - if (str == NULL) { + decoded = decode_str(str, exec_input, tok); + if (decoded == NULL) { PyTokenizer_Free(tok); return NULL; } - /* XXX: constify members. */ - tok->buf = tok->cur = tok->end = tok->inp = (char*)str; + tok->buf = tok->cur = tok->inp = decoded; + tok->end = decoded; return tok; } @@ -743,17 +749,18 @@ struct tok_state * PyTokenizer_FromUTF8(const char *str, int exec_input) { struct tok_state *tok = tok_new(); + char *translated; if (tok == NULL) return NULL; - tok->input = str = translate_newlines(str, exec_input, tok); - if (str == NULL) { + tok->input = translated = translate_newlines(str, exec_input, tok); + if (translated == NULL) { PyTokenizer_Free(tok); return NULL; } tok->decoding_state = STATE_RAW; tok->read_coding_spec = 1; tok->enc = NULL; - tok->str = str; + tok->str = translated; tok->encoding = (char *)PyMem_MALLOC(6); if (!tok->encoding) { PyTokenizer_Free(tok); @@ -761,8 +768,8 @@ PyTokenizer_FromUTF8(const char *str, int exec_input) } strcpy(tok->encoding, "utf-8"); - /* XXX: constify members. */ - tok->buf = tok->cur = tok->end = tok->inp = (char*)str; + tok->buf = tok->cur = tok->inp = translated; + tok->end = translated; return tok; } @@ -812,7 +819,7 @@ PyTokenizer_Free(struct tok_state *tok) if (tok->fp != NULL && tok->buf != NULL) PyMem_FREE(tok->buf); if (tok->input) - PyMem_FREE((char *)tok->input); + PyMem_FREE(tok->input); PyMem_FREE(tok); } @@ -1138,7 +1145,7 @@ tok_decimal_tail(struct tok_state *tok) /* Get next token, after space stripping etc. */ static int -tok_get(struct tok_state *tok, char **p_start, char **p_end) +tok_get(struct tok_state *tok, const char **p_start, const char **p_end) { int c; int blankline, nonascii; @@ -1321,7 +1328,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0])))); if (is_type_ignore) { - *p_start = (char *) ignore_end; + *p_start = ignore_end; *p_end = tok->cur; /* If this type ignore is the only thing on the line, consume the newline also. */ @@ -1331,7 +1338,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) } return TYPE_IGNORE; } else { - *p_start = (char *) type_start; /* after type_comment_prefix */ + *p_start = type_start; /* after type_comment_prefix */ *p_end = tok->cur; return TYPE_COMMENT; } @@ -1410,7 +1417,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) Look ahead one token to see if that is 'def'. */ struct tok_state ahead_tok; - char *ahead_tok_start = NULL, *ahead_tok_end = NULL; + const char *ahead_tok_start = NULL; + const char *ahead_tok_end = NULL; int ahead_tok_kind; memcpy(&ahead_tok, tok, sizeof(ahead_tok)); @@ -1798,7 +1806,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) } int -PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end) +PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end) { int result = tok_get(tok, p_start, p_end); if (tok->decoding_erred) { @@ -1823,7 +1831,9 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) { struct tok_state *tok; FILE *fp; - char *p_start =NULL , *p_end =NULL , *encoding = NULL; + const char *p_start = NULL; + const char *p_end = NULL; + char *encoding = NULL; fd = _Py_dup(fd); if (fd < 0) { diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 92669bfd8a160..5660ea38e9443 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -26,8 +26,8 @@ struct tok_state { char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ - char *end; /* End of input buffer if buf != NULL */ - char *start; /* Start of current token if not NULL */ + const char *end; /* End of input buffer if buf != NULL */ + const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ @@ -60,8 +60,8 @@ struct tok_state { PyObject *decoding_readline; /* open(...).readline */ PyObject *decoding_buffer; const char* enc; /* Encoding for the current str. */ - const char* str; - const char* input; /* Tokenizer's newline translated copy of the string. */ + char* str; + char* input; /* Tokenizer's newline translated copy of the string. */ int type_comments; /* Whether to look for type comments */ @@ -78,7 +78,7 @@ extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*, const char *, const char *); extern void PyTokenizer_Free(struct tok_state *); -extern int PyTokenizer_Get(struct tok_state *, char **, char **); +extern int PyTokenizer_Get(struct tok_state *, const char **, const char **); #define tok_dump _Py_tok_dump

1 0

bpo-39704: Explicitly pass the path to codecov config (GH-18680)
by Ammar Askar 27 Feb '20

27 Feb '20

https://github.com/python/cpython/commit/766b7546a564c8e386a3c31eb06fc1b55e… commit: 766b7546a564c8e386a3c31eb06fc1b55e8f5a25 branch: master author: Ammar Askar <ammar(a)ammaraskar.com> committer: GitHub <noreply(a)github.com> date: 2020-02-27T15:08:30-08:00 summary: bpo-39704: Explicitly pass the path to codecov config (GH-18680) files: M .azure-pipelines/posix-steps.yml M .github/workflows/coverage.yml M .travis.yml diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml index a63659fa20491..b6dde593019e6 100644 --- a/.azure-pipelines/posix-steps.yml +++ b/.azure-pipelines/posix-steps.yml @@ -49,7 +49,7 @@ steps: - script: ./venv/bin/python -m coverage xml displayName: 'Generate coverage.xml' - - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) + - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml displayName: 'Publish code coverage results' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8e1b764ca8df4..bf166ab8f0867 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -65,7 +65,7 @@ jobs: - name: 'Publish code coverage results' run: | source ./.venv/bin/activate - bash <(curl -s https://codecov.io/bash) + bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -84,6 +84,6 @@ jobs: if: always() run: | make pythoninfo - bash <(curl -s https://codecov.io/bash) + bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.travis.yml b/.travis.yml index b1d50d49ca5cc..675e267fe2e82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,7 +94,7 @@ matrix: after_script: # Probably should be after_success once test suite updated to run under coverage.py. # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - source ./venv/bin/activate - - bash <(curl -s https://codecov.io/bash) + - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml - name: "Test code coverage (C)" os: linux language: c @@ -111,7 +111,7 @@ matrix: - xvfb-run make -j4 coverage-report after_script: # Probably should be after_success once test suite updated to run under coverage.py. - make pythoninfo - - bash <(curl -s https://codecov.io/bash) + - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml before_install:

1 0

bpo-39609: set the thread_name_prefix for the default asyncio executor (GH-18458)
by Markus Mohrhard 27 Feb '20

27 Feb '20

https://github.com/python/cpython/commit/374d998b507d34a6c0a3816a163926a8ba… commit: 374d998b507d34a6c0a3816a163926a8ba0c483f branch: master author: Markus Mohrhard <markusm(a)dug.com> committer: GitHub <noreply(a)github.com> date: 2020-02-27T12:01:47-08:00 summary: bpo-39609: set the thread_name_prefix for the default asyncio executor (GH-18458) Just a small debugging improvement to identify the asyncio executor threads. files: A Misc/NEWS.d/next/Library/2020-02-11-19-45-31.bpo-39609.dk40Uw.rst M Lib/asyncio/base_events.py diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index d78724b015370..b2d446a51fedb 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -806,7 +806,9 @@ def run_in_executor(self, executor, func, *args): # Only check when the default executor is being used self._check_default_executor() if executor is None: - executor = concurrent.futures.ThreadPoolExecutor() + executor = concurrent.futures.ThreadPoolExecutor( + thread_name_prefix='asyncio' + ) self._default_executor = executor return futures.wrap_future( executor.submit(func, *args), loop=self) diff --git a/Misc/NEWS.d/next/Library/2020-02-11-19-45-31.bpo-39609.dk40Uw.rst b/Misc/NEWS.d/next/Library/2020-02-11-19-45-31.bpo-39609.dk40Uw.rst new file mode 100644 index 0000000000000..233fad3e763d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-11-19-45-31.bpo-39609.dk40Uw.rst @@ -0,0 +1 @@ +Add thread_name_prefix to default asyncio executor

1 0

bpo-39087: Optimize PyUnicode_AsUTF8AndSize() (GH-18327)
by Inada Naoki 26 Feb '20

26 Feb '20

https://github.com/python/cpython/commit/02a4d57263a9846de35b0db12763ff9e73… commit: 02a4d57263a9846de35b0db12763ff9e7326f62c branch: master author: Inada Naoki <songofacandy(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-27T13:48:59+09:00 summary: bpo-39087: Optimize PyUnicode_AsUTF8AndSize() (GH-18327) Avoid using temporary bytes object. files: A Misc/NEWS.d/next/Core and Builtins/2020-02-03-21-12-39.bpo-39087.YnbUpL.rst M Objects/stringlib/codecs.h M Objects/unicodeobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-03-21-12-39.bpo-39087.YnbUpL.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-03-21-12-39.bpo-39087.YnbUpL.rst new file mode 100644 index 0000000000000..847f78f5b182e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-03-21-12-39.bpo-39087.YnbUpL.rst @@ -0,0 +1,2 @@ +Optimize :c:func:`PyUnicode_AsUTF8` and :c:func:`PyUnicode_AsUTF8AndSize` +slightly when they need to create internal UTF-8 cache. diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 269a5581f7005..eb42e071751d7 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -256,8 +256,9 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, /* UTF-8 encoder specialized for a Unicode kind to avoid the slow PyUnicode_READ() macro. Delete some parts of the code depending on the kind: UCS-1 strings don't need to handle surrogates for example. */ -Py_LOCAL_INLINE(PyObject *) -STRINGLIB(utf8_encoder)(PyObject *unicode, +Py_LOCAL_INLINE(char *) +STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, + PyObject *unicode, STRINGLIB_CHAR *data, Py_ssize_t size, _Py_error_handler error_handler, @@ -277,17 +278,16 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #else /* STRINGLIB_SIZEOF_CHAR == 4 */ const Py_ssize_t max_char_size = 4; #endif - _PyBytesWriter writer; assert(size >= 0); - _PyBytesWriter_Init(&writer); - if (size > PY_SSIZE_T_MAX / max_char_size) { /* integer overflow */ - return PyErr_NoMemory(); + PyErr_NoMemory(); + return NULL; } - p = _PyBytesWriter_Alloc(&writer, size * max_char_size); + _PyBytesWriter_Init(writer); + p = _PyBytesWriter_Alloc(writer, size * max_char_size); if (p == NULL) return NULL; @@ -323,7 +323,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, endpos++; /* Only overallocate the buffer if it's not the last write */ - writer.overallocate = (endpos < size); + writer->overallocate = (endpos < size); switch (error_handler) { @@ -347,8 +347,8 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, case _Py_ERROR_BACKSLASHREPLACE: /* subtract preallocated bytes */ - writer.min_size -= max_char_size * (endpos - startpos); - p = backslashreplace(&writer, p, + writer->min_size -= max_char_size * (endpos - startpos); + p = backslashreplace(writer, p, unicode, startpos, endpos); if (p == NULL) goto error; @@ -357,8 +357,8 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, case _Py_ERROR_XMLCHARREFREPLACE: /* subtract preallocated bytes */ - writer.min_size -= max_char_size * (endpos - startpos); - p = xmlcharrefreplace(&writer, p, + writer->min_size -= max_char_size * (endpos - startpos); + p = xmlcharrefreplace(writer, p, unicode, startpos, endpos); if (p == NULL) goto error; @@ -387,10 +387,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, goto error; /* subtract preallocated bytes */ - writer.min_size -= max_char_size * (newpos - startpos); + writer->min_size -= max_char_size * (newpos - startpos); if (PyBytes_Check(rep)) { - p = _PyBytesWriter_WriteBytes(&writer, p, + p = _PyBytesWriter_WriteBytes(writer, p, PyBytes_AS_STRING(rep), PyBytes_GET_SIZE(rep)); } @@ -406,7 +406,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, goto error; } - p = _PyBytesWriter_WriteBytes(&writer, p, + p = _PyBytesWriter_WriteBytes(writer, p, PyUnicode_DATA(rep), PyUnicode_GET_LENGTH(rep)); } @@ -420,7 +420,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, /* If overallocation was disabled, ensure that it was the last write. Otherwise, we missed an optimization */ - assert(writer.overallocate || i == size); + assert(writer->overallocate || i == size); } else #if STRINGLIB_SIZEOF_CHAR > 2 @@ -449,14 +449,13 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, Py_XDECREF(error_handler_obj); Py_XDECREF(exc); #endif - return _PyBytesWriter_Finish(&writer, p); + return p; #if STRINGLIB_SIZEOF_CHAR > 1 error: Py_XDECREF(rep); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - _PyBytesWriter_Dealloc(&writer); return NULL; #endif } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index ee6d3dfd3945b..e0a666f70da36 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3991,11 +3991,11 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) } +static int unicode_fill_utf8(PyObject *unicode); + const char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { - PyObject *bytes; - if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; @@ -4004,21 +4004,9 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) return NULL; if (PyUnicode_UTF8(unicode) == NULL) { - assert(!PyUnicode_IS_COMPACT_ASCII(unicode)); - bytes = _PyUnicode_AsUTF8String(unicode, NULL); - if (bytes == NULL) - return NULL; - _PyUnicode_UTF8(unicode) = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1); - if (_PyUnicode_UTF8(unicode) == NULL) { - PyErr_NoMemory(); - Py_DECREF(bytes); + if (unicode_fill_utf8(unicode) == -1) { return NULL; } - _PyUnicode_UTF8_LENGTH(unicode) = PyBytes_GET_SIZE(bytes); - memcpy(_PyUnicode_UTF8(unicode), - PyBytes_AS_STRING(bytes), - _PyUnicode_UTF8_LENGTH(unicode) + 1); - Py_DECREF(bytes); } if (psize) @@ -5381,10 +5369,6 @@ static PyObject * unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, const char *errors) { - enum PyUnicode_Kind kind; - void *data; - Py_ssize_t size; - if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; @@ -5397,9 +5381,12 @@ unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, return PyBytes_FromStringAndSize(PyUnicode_UTF8(unicode), PyUnicode_UTF8_LENGTH(unicode)); - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - size = PyUnicode_GET_LENGTH(unicode); + enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + void *data = PyUnicode_DATA(unicode); + Py_ssize_t size = PyUnicode_GET_LENGTH(unicode); + + _PyBytesWriter writer; + char *end; switch (kind) { default: @@ -5407,12 +5394,73 @@ unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, case PyUnicode_1BYTE_KIND: /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ assert(!PyUnicode_IS_ASCII(unicode)); - return ucs1lib_utf8_encoder(unicode, data, size, error_handler, errors); + end = ucs1lib_utf8_encoder(&writer, unicode, data, size, error_handler, errors); + break; + case PyUnicode_2BYTE_KIND: + end = ucs2lib_utf8_encoder(&writer, unicode, data, size, error_handler, errors); + break; + case PyUnicode_4BYTE_KIND: + end = ucs4lib_utf8_encoder(&writer, unicode, data, size, error_handler, errors); + break; + } + + if (end == NULL) { + _PyBytesWriter_Dealloc(&writer); + return NULL; + } + return _PyBytesWriter_Finish(&writer, end); +} + +static int +unicode_fill_utf8(PyObject *unicode) +{ + /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ + assert(!PyUnicode_IS_ASCII(unicode)); + + enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + void *data = PyUnicode_DATA(unicode); + Py_ssize_t size = PyUnicode_GET_LENGTH(unicode); + + _PyBytesWriter writer; + char *end; + + switch (kind) { + default: + Py_UNREACHABLE(); + case PyUnicode_1BYTE_KIND: + end = ucs1lib_utf8_encoder(&writer, unicode, data, size, + _Py_ERROR_STRICT, NULL); + break; case PyUnicode_2BYTE_KIND: - return ucs2lib_utf8_encoder(unicode, data, size, error_handler, errors); + end = ucs2lib_utf8_encoder(&writer, unicode, data, size, + _Py_ERROR_STRICT, NULL); + break; case PyUnicode_4BYTE_KIND: - return ucs4lib_utf8_encoder(unicode, data, size, error_handler, errors); + end = ucs4lib_utf8_encoder(&writer, unicode, data, size, + _Py_ERROR_STRICT, NULL); + break; + } + if (end == NULL) { + _PyBytesWriter_Dealloc(&writer); + return -1; + } + + char *start = writer.use_small_buffer ? writer.small_buffer : + PyBytes_AS_STRING(writer.buffer); + Py_ssize_t len = end - start; + + char *cache = PyObject_MALLOC(len + 1); + if (cache == NULL) { + _PyBytesWriter_Dealloc(&writer); + PyErr_NoMemory(); + return -1; } + _PyUnicode_UTF8(unicode) = cache; + _PyUnicode_UTF8_LENGTH(unicode) = len; + memcpy(cache, start, len); + cache[len] = '\0'; + _PyBytesWriter_Dealloc(&writer); + return 0; } PyObject *

1 0